From 616a283cbf54fdb7fbc32563da1c881af7967c0d Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 10 Dec 2023 14:07:06 -0600 Subject: [PATCH] Extract library and refactor (#7) * add a library for splam detection well tested and somewhat documented, not wired in yet * add a section about library to readme * integrate new library * integrate locator * update docs * limit the number of events in a short time period reload ham and spam in watcher tests * add test for events debouncing * drop legacy deps from bot to updaters this is part of detector now * add test for forwarded ban, drop dead code * add watch interval parameter * minor logging improvements * minor simplifications * update comments * extend logging * fix test * convert server to chi, add middlewares for safety * document code, use common terms * update lib docs * rename to make it more consistent, add events package comment * drop separate super user file, add code comments * remote deployment on tags only --- .github/workflows/ci.yml | 3 +- Makefile | 2 +- README.md | 43 +- app/bot/bot.go | 5 + app/bot/file_watcher.go | 111 - app/bot/file_watcher_test.go | 138 - app/bot/mocks/detector.go | 391 +++ app/bot/sample_updater.go | 16 +- app/bot/sample_updater_test.go | 6 +- app/bot/spam.go | 540 +--- app/bot/spam_test.go | 570 ++-- app/bot/testdata/ham-samples.txt | 2 - app/bot/testdata/spam-exclude-token.txt | 33 - app/bot/testdata/spam-samples.txt | 2 - app/bot/testdata/stop-words.txt | 9 - app/events/{telegram.go => events.go} | 147 +- .../{telegram_test.go => events_test.go} | 64 +- app/events/locator.go | 76 + app/events/locator_test.go | 89 + app/events/superuser.go | 18 - app/events/superuser_test.go | 47 - app/main.go | 101 +- app/main_test.go | 4 +- app/server/server.go | 106 +- app/server/server_test.go | 60 +- data/spam-samples.txt | 1 + go.mod | 15 +- go.sum | 39 +- {app/bot => lib}/classifier.go | 8 +- {app/bot => lib}/classifier_test.go | 2 +- lib/detector.go | 433 +++ lib/detector_test.go | 411 +++ lib/emoji.go | 14 + lib/emoji_test.go | 55 + lib/lib.go | 44 + {app/bot => lib}/mocks/http_client.go | 16 +- lib/mocks/sample_updater.go | 108 + site/docs/index.md | 49 +- .../aymerick/douceur/css/declaration.go | 60 - .../github.com/aymerick/douceur/css/rule.go | 230 -- .../aymerick/douceur/css/stylesheet.go | 25 - .../aymerick/douceur/parser/parser.go | 409 --- .../github.com/didip/tollbooth/v7/.gitignore | 3 + .../didip/tollbooth/v7/.golangci.yml | 37 + .../douceur => didip/tollbooth/v7}/LICENSE | 11 +- .../github.com/didip/tollbooth/v7/README.md | 184 ++ .../didip/tollbooth/v7/errors/errors.go | 15 + .../didip/tollbooth/v7/internal/time/AUTHORS | 3 + .../tollbooth/v7/internal/time/CONTRIBUTORS | 3 + .../didip/tollbooth/v7/internal/time}/LICENSE | 0 .../didip/tollbooth/v7/internal/time}/PATENTS | 0 .../tollbooth/v7/internal/time/rate/rate.go | 396 +++ .../didip/tollbooth/v7/libstring/libstring.go | 101 + .../didip/tollbooth/v7/limiter/limiter.go | 610 ++++ .../tollbooth/v7/limiter/limiter_options.go | 14 + .../didip/tollbooth/v7/tollbooth.go | 349 +++ .../github.com/didip/tollbooth_chi/README.md | 33 + .../didip/tollbooth_chi/tollbooth_chi.go | 45 + vendor/github.com/go-chi/chi/.gitignore | 3 + vendor/github.com/go-chi/chi/CHANGELOG.md | 269 ++ vendor/github.com/go-chi/chi/CONTRIBUTING.md | 31 + vendor/github.com/go-chi/chi/LICENSE | 20 + vendor/github.com/go-chi/chi/Makefile | 14 + vendor/github.com/go-chi/chi/README.md | 511 ++++ vendor/github.com/go-chi/chi/chain.go | 49 + vendor/github.com/go-chi/chi/chi.go | 134 + vendor/github.com/go-chi/chi/context.go | 157 ++ vendor/github.com/go-chi/chi/mux.go | 479 ++++ vendor/github.com/go-chi/chi/tree.go | 866 ++++++ vendor/github.com/go-chi/chi/v5/.gitignore | 3 + vendor/github.com/go-chi/chi/v5/CHANGELOG.md | 331 +++ .../github.com/go-chi/chi/v5/CONTRIBUTING.md | 31 + vendor/github.com/go-chi/chi/v5/LICENSE | 20 + vendor/github.com/go-chi/chi/v5/Makefile | 22 + vendor/github.com/go-chi/chi/v5/README.md | 500 ++++ vendor/github.com/go-chi/chi/v5/chain.go | 49 + vendor/github.com/go-chi/chi/v5/chi.go | 134 + vendor/github.com/go-chi/chi/v5/context.go | 160 ++ .../go-chi/chi/v5/middleware/basic_auth.go | 33 + .../go-chi/chi/v5/middleware/clean_path.go | 28 + .../go-chi/chi/v5/middleware/compress.go | 403 +++ .../chi/v5/middleware/content_charset.go | 51 + .../chi/v5/middleware/content_encoding.go | 34 + .../go-chi/chi/v5/middleware/content_type.go | 49 + .../go-chi/chi/v5/middleware/get_head.go | 39 + .../go-chi/chi/v5/middleware/heartbeat.go | 26 + .../go-chi/chi/v5/middleware/logger.go | 171 ++ .../go-chi/chi/v5/middleware/maybe.go | 18 + .../go-chi/chi/v5/middleware/middleware.go | 23 + .../go-chi/chi/v5/middleware/nocache.go | 58 + .../go-chi/chi/v5/middleware/page_route.go | 20 + .../go-chi/chi/v5/middleware/path_rewrite.go | 16 + .../go-chi/chi/v5/middleware/profiler.go | 62 + .../go-chi/chi/v5/middleware/realip.go | 60 + .../go-chi/chi/v5/middleware/recoverer.go | 204 ++ .../go-chi/chi/v5/middleware/request_id.go | 96 + .../go-chi/chi/v5/middleware/request_size.go | 18 + .../go-chi/chi/v5/middleware/route_headers.go | 160 ++ .../go-chi/chi/v5/middleware/strip.go | 62 + .../go-chi/chi/v5/middleware/terminal.go | 63 + .../go-chi/chi/v5/middleware/throttle.go | 132 + .../go-chi/chi/v5/middleware/timeout.go | 49 + .../go-chi/chi/v5/middleware/url_format.go | 76 + .../go-chi/chi/v5/middleware/value.go | 17 + .../go-chi/chi/v5/middleware/wrap_writer.go | 219 ++ vendor/github.com/go-chi/chi/v5/mux.go | 493 ++++ vendor/github.com/go-chi/chi/v5/tree.go | 892 ++++++ vendor/github.com/go-pkgz/email/.golangci.yml | 57 - vendor/github.com/go-pkgz/email/LICENSE | 21 - vendor/github.com/go-pkgz/email/README.md | 71 - vendor/github.com/go-pkgz/email/auth.go | 60 - vendor/github.com/go-pkgz/email/email.go | 366 --- vendor/github.com/go-pkgz/email/options.go | 77 - .../{email => expirable-cache}/.gitignore | 0 .../.golangci.yml | 24 +- .../{notify => expirable-cache}/LICENSE | 3 +- .../go-pkgz/expirable-cache/README.md | 70 + .../go-pkgz/expirable-cache/cache.go | 272 ++ .../go-pkgz/expirable-cache/options.go | 40 + vendor/github.com/go-pkgz/notify/README.md | 202 -- vendor/github.com/go-pkgz/notify/email.go | 146 - vendor/github.com/go-pkgz/notify/interface.go | 28 - vendor/github.com/go-pkgz/notify/slack.go | 108 - vendor/github.com/go-pkgz/notify/telegram.go | 479 ---- vendor/github.com/go-pkgz/notify/webhook.go | 96 - .../github.com/go-pkgz/repeater/.travis.yml | 20 - vendor/github.com/go-pkgz/repeater/README.md | 35 - .../github.com/go-pkgz/repeater/repeater.go | 64 - .../go-pkgz/repeater/strategy/backoff.go | 59 - .../go-pkgz/repeater/strategy/fixed.go | 36 - .../go-pkgz/repeater/strategy/strategy.go | 35 - .../go-pkgz/{repeater => rest}/.gitignore | 1 + .../go-pkgz/{notify => rest}/.golangci.yml | 33 +- .../go-pkgz/{repeater => rest}/LICENSE | 2 +- vendor/github.com/go-pkgz/rest/README.md | 244 ++ vendor/github.com/go-pkgz/rest/basic_auth.go | 73 + vendor/github.com/go-pkgz/rest/benchmarks.go | 159 ++ vendor/github.com/go-pkgz/rest/blackwords.go | 39 + .../github.com/go-pkgz/rest/cache_control.go | 62 + .../github.com/go-pkgz/rest/depricattion.go | 21 + vendor/github.com/go-pkgz/rest/file_server.go | 185 ++ vendor/github.com/go-pkgz/rest/gzip.go | 88 + vendor/github.com/go-pkgz/rest/httperrors.go | 67 + .../github.com/go-pkgz/rest/logger/logger.go | 359 +++ .../github.com/go-pkgz/rest/logger/options.go | 63 + vendor/github.com/go-pkgz/rest/metrics.go | 28 + vendor/github.com/go-pkgz/rest/middleware.go | 184 ++ vendor/github.com/go-pkgz/rest/nocache.go | 55 + vendor/github.com/go-pkgz/rest/onlyfrom.go | 59 + vendor/github.com/go-pkgz/rest/profiler.go | 48 + vendor/github.com/go-pkgz/rest/realip/real.go | 81 + vendor/github.com/go-pkgz/rest/rest.go | 99 + vendor/github.com/go-pkgz/rest/rewrite.go | 57 + vendor/github.com/go-pkgz/rest/sizelimit.go | 40 + vendor/github.com/go-pkgz/rest/throttle.go | 46 + vendor/github.com/go-pkgz/rest/trace.go | 48 + vendor/github.com/gorilla/css/LICENSE | 28 - vendor/github.com/gorilla/css/scanner/doc.go | 33 - .../github.com/gorilla/css/scanner/scanner.go | 360 --- .../gorilla/websocket/.editorconfig | 20 - .../github.com/gorilla/websocket/.gitignore | 1 - .../gorilla/websocket/.golangci.yml | 3 - vendor/github.com/gorilla/websocket/LICENSE | 27 - vendor/github.com/gorilla/websocket/Makefile | 34 - vendor/github.com/gorilla/websocket/README.md | 36 - vendor/github.com/gorilla/websocket/client.go | 444 --- .../gorilla/websocket/compression.go | 153 - vendor/github.com/gorilla/websocket/conn.go | 1267 --------- vendor/github.com/gorilla/websocket/doc.go | 227 -- vendor/github.com/gorilla/websocket/join.go | 42 - vendor/github.com/gorilla/websocket/json.go | 60 - vendor/github.com/gorilla/websocket/mask.go | 59 - .../github.com/gorilla/websocket/mask_safe.go | 16 - .../github.com/gorilla/websocket/prepared.go | 102 - vendor/github.com/gorilla/websocket/proxy.go | 86 - vendor/github.com/gorilla/websocket/server.go | 389 --- .../gorilla/websocket/tls_handshake.go | 18 - vendor/github.com/gorilla/websocket/util.go | 298 -- .../microcosm-cc/bluemonday/.coveralls.yml | 1 - .../microcosm-cc/bluemonday/.editorconfig | 4 - .../microcosm-cc/bluemonday/.gitattributes | 1 - .../microcosm-cc/bluemonday/.gitignore | 15 - .../microcosm-cc/bluemonday/.travis.yml | 26 - .../microcosm-cc/bluemonday/CONTRIBUTING.md | 51 - .../microcosm-cc/bluemonday/CREDITS.md | 8 - .../microcosm-cc/bluemonday/LICENSE.md | 31 - .../microcosm-cc/bluemonday/Makefile | 48 - .../microcosm-cc/bluemonday/README.md | 418 --- .../microcosm-cc/bluemonday/SECURITY.md | 15 - .../microcosm-cc/bluemonday/css/handlers.go | 2016 -------------- .../github.com/microcosm-cc/bluemonday/doc.go | 104 - .../microcosm-cc/bluemonday/helpers.go | 300 -- .../microcosm-cc/bluemonday/policies.go | 253 -- .../microcosm-cc/bluemonday/policy.go | 990 ------- .../microcosm-cc/bluemonday/sanitize.go | 1089 -------- .../bluemonday/stringwriterwriter_go1.12.go | 11 - .../bluemonday/stringwriterwriter_ltgo1.12.go | 15 - vendor/github.com/slack-go/slack/.gitignore | 3 - .../github.com/slack-go/slack/.golangci.yml | 14 - vendor/github.com/slack-go/slack/CHANGELOG.md | 103 - vendor/github.com/slack-go/slack/LICENSE | 23 - vendor/github.com/slack-go/slack/Makefile | 36 - vendor/github.com/slack-go/slack/README.md | 111 - vendor/github.com/slack-go/slack/TODO.txt | 3 - vendor/github.com/slack-go/slack/admin.go | 207 -- vendor/github.com/slack-go/slack/apps.go | 64 - .../github.com/slack-go/slack/attachments.go | 98 - vendor/github.com/slack-go/slack/audit.go | 152 - vendor/github.com/slack-go/slack/auth.go | 74 - vendor/github.com/slack-go/slack/block.go | 82 - .../github.com/slack-go/slack/block_action.go | 26 - .../slack-go/slack/block_context.go | 32 - .../github.com/slack-go/slack/block_conv.go | 437 --- .../slack-go/slack/block_divider.go | 22 - .../slack-go/slack/block_element.go | 593 ---- .../github.com/slack-go/slack/block_file.go | 26 - .../github.com/slack-go/slack/block_header.go | 38 - .../github.com/slack-go/slack/block_image.go | 28 - .../github.com/slack-go/slack/block_input.go | 30 - .../github.com/slack-go/slack/block_object.go | 248 -- .../slack-go/slack/block_rich_text.go | 383 --- .../slack-go/slack/block_section.go | 42 - .../slack-go/slack/block_unknown.go | 13 - vendor/github.com/slack-go/slack/bookmarks.go | 159 -- vendor/github.com/slack-go/slack/bots.go | 58 - vendor/github.com/slack-go/slack/channels.go | 36 - vendor/github.com/slack-go/slack/chat.go | 876 ------ vendor/github.com/slack-go/slack/comment.go | 10 - .../github.com/slack-go/slack/conversation.go | 739 ----- vendor/github.com/slack-go/slack/dialog.go | 120 - .../slack-go/slack/dialog_select.go | 115 - .../github.com/slack-go/slack/dialog_text.go | 59 - vendor/github.com/slack-go/slack/dnd.go | 151 - vendor/github.com/slack-go/slack/emoji.go | 35 - vendor/github.com/slack-go/slack/errors.go | 21 - vendor/github.com/slack-go/slack/files.go | 597 ---- vendor/github.com/slack-go/slack/groups.go | 7 - vendor/github.com/slack-go/slack/history.go | 37 - vendor/github.com/slack-go/slack/im.go | 21 - vendor/github.com/slack-go/slack/info.go | 476 ---- .../github.com/slack-go/slack/interactions.go | 238 -- .../slack/internal/backoff/backoff.go | 62 - .../slack/internal/errorsx/errorsx.go | 17 - .../slack-go/slack/internal/timex/timex.go | 18 - vendor/github.com/slack-go/slack/item.go | 75 - vendor/github.com/slack-go/slack/logger.go | 60 - vendor/github.com/slack-go/slack/logo.png | Bin 52440 -> 0 bytes vendor/github.com/slack-go/slack/manifests.go | 287 -- vendor/github.com/slack-go/slack/messageID.go | 30 - vendor/github.com/slack-go/slack/messages.go | 259 -- vendor/github.com/slack-go/slack/metadata.go | 7 - vendor/github.com/slack-go/slack/misc.go | 349 --- vendor/github.com/slack-go/slack/oauth.go | 158 -- .../github.com/slack-go/slack/pagination.go | 20 - vendor/github.com/slack-go/slack/pins.go | 94 - vendor/github.com/slack-go/slack/reactions.go | 270 -- vendor/github.com/slack-go/slack/reminders.go | 132 - .../github.com/slack-go/slack/remotefiles.go | 316 --- vendor/github.com/slack-go/slack/rtm.go | 131 - vendor/github.com/slack-go/slack/search.go | 156 -- vendor/github.com/slack-go/slack/security.go | 108 - vendor/github.com/slack-go/slack/slack.go | 174 -- .../slack-go/slack/slackutilsx/slackutilsx.go | 64 - vendor/github.com/slack-go/slack/slash.go | 55 - .../github.com/slack-go/slack/socket_mode.go | 34 - vendor/github.com/slack-go/slack/stars.go | 264 -- .../slack-go/slack/status_code_error.go | 28 - vendor/github.com/slack-go/slack/team.go | 236 -- vendor/github.com/slack-go/slack/tokens.go | 50 - .../github.com/slack-go/slack/usergroups.go | 303 -- vendor/github.com/slack-go/slack/users.go | 738 ----- vendor/github.com/slack-go/slack/views.go | 306 -- vendor/github.com/slack-go/slack/webhooks.go | 63 - vendor/github.com/slack-go/slack/websocket.go | 103 - .../slack-go/slack/websocket_channels.go | 72 - .../slack/websocket_desktop_notification.go | 19 - .../github.com/slack-go/slack/websocket_dm.go | 23 - .../slack-go/slack/websocket_dnd.go | 8 - .../slack-go/slack/websocket_files.go | 49 - .../slack-go/slack/websocket_groups.go | 49 - .../slack-go/slack/websocket_internals.go | 102 - .../slack-go/slack/websocket_managed_conn.go | 611 ---- .../slack-go/slack/websocket_misc.go | 141 - .../websocket_mobile_in_app_notification.go | 20 - .../slack-go/slack/websocket_pins.go | 16 - .../slack-go/slack/websocket_reactions.go | 25 - .../slack-go/slack/websocket_stars.go | 14 - .../slack-go/slack/websocket_subteam.go | 35 - .../slack-go/slack/websocket_teams.go | 33 - .../slack-go/slack/workflow_step.go | 98 - .../slack-go/slack/workflow_step_execute.go | 85 - vendor/golang.org/x/net/html/atom/atom.go | 78 - vendor/golang.org/x/net/html/atom/table.go | 783 ------ vendor/golang.org/x/net/html/const.go | 111 - vendor/golang.org/x/net/html/doc.go | 127 - vendor/golang.org/x/net/html/doctype.go | 156 -- vendor/golang.org/x/net/html/entity.go | 2253 --------------- vendor/golang.org/x/net/html/escape.go | 339 --- vendor/golang.org/x/net/html/foreign.go | 222 -- vendor/golang.org/x/net/html/node.go | 225 -- vendor/golang.org/x/net/html/parse.go | 2460 ----------------- vendor/golang.org/x/net/html/render.go | 293 -- vendor/golang.org/x/net/html/token.go | 1268 --------- .../golang.org/x/net/internal/socks/client.go | 168 -- .../golang.org/x/net/internal/socks/socks.go | 317 --- vendor/golang.org/x/net/proxy/dial.go | 54 - vendor/golang.org/x/net/proxy/direct.go | 31 - vendor/golang.org/x/net/proxy/per_host.go | 155 -- vendor/golang.org/x/net/proxy/proxy.go | 149 - vendor/golang.org/x/net/proxy/socks5.go | 42 - vendor/modules.txt | 62 +- 311 files changed, 14196 insertions(+), 35377 deletions(-) delete mode 100644 app/bot/file_watcher.go delete mode 100644 app/bot/file_watcher_test.go create mode 100644 app/bot/mocks/detector.go delete mode 100644 app/bot/testdata/ham-samples.txt delete mode 100644 app/bot/testdata/spam-exclude-token.txt delete mode 100644 app/bot/testdata/spam-samples.txt delete mode 100644 app/bot/testdata/stop-words.txt rename app/events/{telegram.go => events.go} (75%) rename app/events/{telegram_test.go => events_test.go} (90%) create mode 100644 app/events/locator.go create mode 100644 app/events/locator_test.go delete mode 100644 app/events/superuser.go delete mode 100644 app/events/superuser_test.go rename {app/bot => lib}/classifier.go (99%) rename {app/bot => lib}/classifier_test.go (99%) create mode 100644 lib/detector.go create mode 100644 lib/detector_test.go create mode 100644 lib/emoji.go create mode 100644 lib/emoji_test.go create mode 100644 lib/lib.go rename {app/bot => lib}/mocks/http_client.go (71%) create mode 100644 lib/mocks/sample_updater.go delete mode 100644 vendor/github.com/aymerick/douceur/css/declaration.go delete mode 100644 vendor/github.com/aymerick/douceur/css/rule.go delete mode 100644 vendor/github.com/aymerick/douceur/css/stylesheet.go delete mode 100644 vendor/github.com/aymerick/douceur/parser/parser.go create mode 100644 vendor/github.com/didip/tollbooth/v7/.gitignore create mode 100644 vendor/github.com/didip/tollbooth/v7/.golangci.yml rename vendor/github.com/{aymerick/douceur => didip/tollbooth/v7}/LICENSE (88%) create mode 100644 vendor/github.com/didip/tollbooth/v7/README.md create mode 100644 vendor/github.com/didip/tollbooth/v7/errors/errors.go create mode 100644 vendor/github.com/didip/tollbooth/v7/internal/time/AUTHORS create mode 100644 vendor/github.com/didip/tollbooth/v7/internal/time/CONTRIBUTORS rename vendor/{golang.org/x/net => github.com/didip/tollbooth/v7/internal/time}/LICENSE (100%) rename vendor/{golang.org/x/net => github.com/didip/tollbooth/v7/internal/time}/PATENTS (100%) create mode 100644 vendor/github.com/didip/tollbooth/v7/internal/time/rate/rate.go create mode 100644 vendor/github.com/didip/tollbooth/v7/libstring/libstring.go create mode 100644 vendor/github.com/didip/tollbooth/v7/limiter/limiter.go create mode 100644 vendor/github.com/didip/tollbooth/v7/limiter/limiter_options.go create mode 100644 vendor/github.com/didip/tollbooth/v7/tollbooth.go create mode 100644 vendor/github.com/didip/tollbooth_chi/README.md create mode 100644 vendor/github.com/didip/tollbooth_chi/tollbooth_chi.go create mode 100644 vendor/github.com/go-chi/chi/.gitignore create mode 100644 vendor/github.com/go-chi/chi/CHANGELOG.md create mode 100644 vendor/github.com/go-chi/chi/CONTRIBUTING.md create mode 100644 vendor/github.com/go-chi/chi/LICENSE create mode 100644 vendor/github.com/go-chi/chi/Makefile create mode 100644 vendor/github.com/go-chi/chi/README.md create mode 100644 vendor/github.com/go-chi/chi/chain.go create mode 100644 vendor/github.com/go-chi/chi/chi.go create mode 100644 vendor/github.com/go-chi/chi/context.go create mode 100644 vendor/github.com/go-chi/chi/mux.go create mode 100644 vendor/github.com/go-chi/chi/tree.go create mode 100644 vendor/github.com/go-chi/chi/v5/.gitignore create mode 100644 vendor/github.com/go-chi/chi/v5/CHANGELOG.md create mode 100644 vendor/github.com/go-chi/chi/v5/CONTRIBUTING.md create mode 100644 vendor/github.com/go-chi/chi/v5/LICENSE create mode 100644 vendor/github.com/go-chi/chi/v5/Makefile create mode 100644 vendor/github.com/go-chi/chi/v5/README.md create mode 100644 vendor/github.com/go-chi/chi/v5/chain.go create mode 100644 vendor/github.com/go-chi/chi/v5/chi.go create mode 100644 vendor/github.com/go-chi/chi/v5/context.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/basic_auth.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/clean_path.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/compress.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/content_charset.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/content_encoding.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/content_type.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/get_head.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/heartbeat.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/logger.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/maybe.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/middleware.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/nocache.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/page_route.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/path_rewrite.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/profiler.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/realip.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/recoverer.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/request_id.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/request_size.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/route_headers.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/strip.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/terminal.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/throttle.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/timeout.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/url_format.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/value.go create mode 100644 vendor/github.com/go-chi/chi/v5/middleware/wrap_writer.go create mode 100644 vendor/github.com/go-chi/chi/v5/mux.go create mode 100644 vendor/github.com/go-chi/chi/v5/tree.go delete mode 100644 vendor/github.com/go-pkgz/email/.golangci.yml delete mode 100644 vendor/github.com/go-pkgz/email/LICENSE delete mode 100644 vendor/github.com/go-pkgz/email/README.md delete mode 100644 vendor/github.com/go-pkgz/email/auth.go delete mode 100644 vendor/github.com/go-pkgz/email/email.go delete mode 100644 vendor/github.com/go-pkgz/email/options.go rename vendor/github.com/go-pkgz/{email => expirable-cache}/.gitignore (100%) rename vendor/github.com/go-pkgz/{repeater => expirable-cache}/.golangci.yml (71%) rename vendor/github.com/go-pkgz/{notify => expirable-cache}/LICENSE (94%) create mode 100644 vendor/github.com/go-pkgz/expirable-cache/README.md create mode 100644 vendor/github.com/go-pkgz/expirable-cache/cache.go create mode 100644 vendor/github.com/go-pkgz/expirable-cache/options.go delete mode 100644 vendor/github.com/go-pkgz/notify/README.md delete mode 100644 vendor/github.com/go-pkgz/notify/email.go delete mode 100644 vendor/github.com/go-pkgz/notify/interface.go delete mode 100644 vendor/github.com/go-pkgz/notify/slack.go delete mode 100644 vendor/github.com/go-pkgz/notify/telegram.go delete mode 100644 vendor/github.com/go-pkgz/notify/webhook.go delete mode 100644 vendor/github.com/go-pkgz/repeater/.travis.yml delete mode 100644 vendor/github.com/go-pkgz/repeater/README.md delete mode 100644 vendor/github.com/go-pkgz/repeater/repeater.go delete mode 100644 vendor/github.com/go-pkgz/repeater/strategy/backoff.go delete mode 100644 vendor/github.com/go-pkgz/repeater/strategy/fixed.go delete mode 100644 vendor/github.com/go-pkgz/repeater/strategy/strategy.go rename vendor/github.com/go-pkgz/{repeater => rest}/.gitignore (96%) rename vendor/github.com/go-pkgz/{notify => rest}/.golangci.yml (74%) rename vendor/github.com/go-pkgz/{repeater => rest}/LICENSE (97%) create mode 100644 vendor/github.com/go-pkgz/rest/README.md create mode 100644 vendor/github.com/go-pkgz/rest/basic_auth.go create mode 100644 vendor/github.com/go-pkgz/rest/benchmarks.go create mode 100644 vendor/github.com/go-pkgz/rest/blackwords.go create mode 100644 vendor/github.com/go-pkgz/rest/cache_control.go create mode 100644 vendor/github.com/go-pkgz/rest/depricattion.go create mode 100644 vendor/github.com/go-pkgz/rest/file_server.go create mode 100644 vendor/github.com/go-pkgz/rest/gzip.go create mode 100644 vendor/github.com/go-pkgz/rest/httperrors.go create mode 100644 vendor/github.com/go-pkgz/rest/logger/logger.go create mode 100644 vendor/github.com/go-pkgz/rest/logger/options.go create mode 100644 vendor/github.com/go-pkgz/rest/metrics.go create mode 100644 vendor/github.com/go-pkgz/rest/middleware.go create mode 100644 vendor/github.com/go-pkgz/rest/nocache.go create mode 100644 vendor/github.com/go-pkgz/rest/onlyfrom.go create mode 100644 vendor/github.com/go-pkgz/rest/profiler.go create mode 100644 vendor/github.com/go-pkgz/rest/realip/real.go create mode 100644 vendor/github.com/go-pkgz/rest/rest.go create mode 100644 vendor/github.com/go-pkgz/rest/rewrite.go create mode 100644 vendor/github.com/go-pkgz/rest/sizelimit.go create mode 100644 vendor/github.com/go-pkgz/rest/throttle.go create mode 100644 vendor/github.com/go-pkgz/rest/trace.go delete mode 100644 vendor/github.com/gorilla/css/LICENSE delete mode 100644 vendor/github.com/gorilla/css/scanner/doc.go delete mode 100644 vendor/github.com/gorilla/css/scanner/scanner.go delete mode 100644 vendor/github.com/gorilla/websocket/.editorconfig delete mode 100644 vendor/github.com/gorilla/websocket/.gitignore delete mode 100644 vendor/github.com/gorilla/websocket/.golangci.yml delete mode 100644 vendor/github.com/gorilla/websocket/LICENSE delete mode 100644 vendor/github.com/gorilla/websocket/Makefile delete mode 100644 vendor/github.com/gorilla/websocket/README.md delete mode 100644 vendor/github.com/gorilla/websocket/client.go delete mode 100644 vendor/github.com/gorilla/websocket/compression.go delete mode 100644 vendor/github.com/gorilla/websocket/conn.go delete mode 100644 vendor/github.com/gorilla/websocket/doc.go delete mode 100644 vendor/github.com/gorilla/websocket/join.go delete mode 100644 vendor/github.com/gorilla/websocket/json.go delete mode 100644 vendor/github.com/gorilla/websocket/mask.go delete mode 100644 vendor/github.com/gorilla/websocket/mask_safe.go delete mode 100644 vendor/github.com/gorilla/websocket/prepared.go delete mode 100644 vendor/github.com/gorilla/websocket/proxy.go delete mode 100644 vendor/github.com/gorilla/websocket/server.go delete mode 100644 vendor/github.com/gorilla/websocket/tls_handshake.go delete mode 100644 vendor/github.com/gorilla/websocket/util.go delete mode 100644 vendor/github.com/microcosm-cc/bluemonday/.coveralls.yml delete mode 100644 vendor/github.com/microcosm-cc/bluemonday/.editorconfig delete mode 100644 vendor/github.com/microcosm-cc/bluemonday/.gitattributes delete mode 100644 vendor/github.com/microcosm-cc/bluemonday/.gitignore delete mode 100644 vendor/github.com/microcosm-cc/bluemonday/.travis.yml delete mode 100644 vendor/github.com/microcosm-cc/bluemonday/CONTRIBUTING.md delete mode 100644 vendor/github.com/microcosm-cc/bluemonday/CREDITS.md delete mode 100644 vendor/github.com/microcosm-cc/bluemonday/LICENSE.md delete mode 100644 vendor/github.com/microcosm-cc/bluemonday/Makefile delete mode 100644 vendor/github.com/microcosm-cc/bluemonday/README.md delete mode 100644 vendor/github.com/microcosm-cc/bluemonday/SECURITY.md delete mode 100644 vendor/github.com/microcosm-cc/bluemonday/css/handlers.go delete mode 100644 vendor/github.com/microcosm-cc/bluemonday/doc.go delete mode 100644 vendor/github.com/microcosm-cc/bluemonday/helpers.go delete mode 100644 vendor/github.com/microcosm-cc/bluemonday/policies.go delete mode 100644 vendor/github.com/microcosm-cc/bluemonday/policy.go delete mode 100644 vendor/github.com/microcosm-cc/bluemonday/sanitize.go delete mode 100644 vendor/github.com/microcosm-cc/bluemonday/stringwriterwriter_go1.12.go delete mode 100644 vendor/github.com/microcosm-cc/bluemonday/stringwriterwriter_ltgo1.12.go delete mode 100644 vendor/github.com/slack-go/slack/.gitignore delete mode 100644 vendor/github.com/slack-go/slack/.golangci.yml delete mode 100644 vendor/github.com/slack-go/slack/CHANGELOG.md delete mode 100644 vendor/github.com/slack-go/slack/LICENSE delete mode 100644 vendor/github.com/slack-go/slack/Makefile delete mode 100644 vendor/github.com/slack-go/slack/README.md delete mode 100644 vendor/github.com/slack-go/slack/TODO.txt delete mode 100644 vendor/github.com/slack-go/slack/admin.go delete mode 100644 vendor/github.com/slack-go/slack/apps.go delete mode 100644 vendor/github.com/slack-go/slack/attachments.go delete mode 100644 vendor/github.com/slack-go/slack/audit.go delete mode 100644 vendor/github.com/slack-go/slack/auth.go delete mode 100644 vendor/github.com/slack-go/slack/block.go delete mode 100644 vendor/github.com/slack-go/slack/block_action.go delete mode 100644 vendor/github.com/slack-go/slack/block_context.go delete mode 100644 vendor/github.com/slack-go/slack/block_conv.go delete mode 100644 vendor/github.com/slack-go/slack/block_divider.go delete mode 100644 vendor/github.com/slack-go/slack/block_element.go delete mode 100644 vendor/github.com/slack-go/slack/block_file.go delete mode 100644 vendor/github.com/slack-go/slack/block_header.go delete mode 100644 vendor/github.com/slack-go/slack/block_image.go delete mode 100644 vendor/github.com/slack-go/slack/block_input.go delete mode 100644 vendor/github.com/slack-go/slack/block_object.go delete mode 100644 vendor/github.com/slack-go/slack/block_rich_text.go delete mode 100644 vendor/github.com/slack-go/slack/block_section.go delete mode 100644 vendor/github.com/slack-go/slack/block_unknown.go delete mode 100644 vendor/github.com/slack-go/slack/bookmarks.go delete mode 100644 vendor/github.com/slack-go/slack/bots.go delete mode 100644 vendor/github.com/slack-go/slack/channels.go delete mode 100644 vendor/github.com/slack-go/slack/chat.go delete mode 100644 vendor/github.com/slack-go/slack/comment.go delete mode 100644 vendor/github.com/slack-go/slack/conversation.go delete mode 100644 vendor/github.com/slack-go/slack/dialog.go delete mode 100644 vendor/github.com/slack-go/slack/dialog_select.go delete mode 100644 vendor/github.com/slack-go/slack/dialog_text.go delete mode 100644 vendor/github.com/slack-go/slack/dnd.go delete mode 100644 vendor/github.com/slack-go/slack/emoji.go delete mode 100644 vendor/github.com/slack-go/slack/errors.go delete mode 100644 vendor/github.com/slack-go/slack/files.go delete mode 100644 vendor/github.com/slack-go/slack/groups.go delete mode 100644 vendor/github.com/slack-go/slack/history.go delete mode 100644 vendor/github.com/slack-go/slack/im.go delete mode 100644 vendor/github.com/slack-go/slack/info.go delete mode 100644 vendor/github.com/slack-go/slack/interactions.go delete mode 100644 vendor/github.com/slack-go/slack/internal/backoff/backoff.go delete mode 100644 vendor/github.com/slack-go/slack/internal/errorsx/errorsx.go delete mode 100644 vendor/github.com/slack-go/slack/internal/timex/timex.go delete mode 100644 vendor/github.com/slack-go/slack/item.go delete mode 100644 vendor/github.com/slack-go/slack/logger.go delete mode 100644 vendor/github.com/slack-go/slack/logo.png delete mode 100644 vendor/github.com/slack-go/slack/manifests.go delete mode 100644 vendor/github.com/slack-go/slack/messageID.go delete mode 100644 vendor/github.com/slack-go/slack/messages.go delete mode 100644 vendor/github.com/slack-go/slack/metadata.go delete mode 100644 vendor/github.com/slack-go/slack/misc.go delete mode 100644 vendor/github.com/slack-go/slack/oauth.go delete mode 100644 vendor/github.com/slack-go/slack/pagination.go delete mode 100644 vendor/github.com/slack-go/slack/pins.go delete mode 100644 vendor/github.com/slack-go/slack/reactions.go delete mode 100644 vendor/github.com/slack-go/slack/reminders.go delete mode 100644 vendor/github.com/slack-go/slack/remotefiles.go delete mode 100644 vendor/github.com/slack-go/slack/rtm.go delete mode 100644 vendor/github.com/slack-go/slack/search.go delete mode 100644 vendor/github.com/slack-go/slack/security.go delete mode 100644 vendor/github.com/slack-go/slack/slack.go delete mode 100644 vendor/github.com/slack-go/slack/slackutilsx/slackutilsx.go delete mode 100644 vendor/github.com/slack-go/slack/slash.go delete mode 100644 vendor/github.com/slack-go/slack/socket_mode.go delete mode 100644 vendor/github.com/slack-go/slack/stars.go delete mode 100644 vendor/github.com/slack-go/slack/status_code_error.go delete mode 100644 vendor/github.com/slack-go/slack/team.go delete mode 100644 vendor/github.com/slack-go/slack/tokens.go delete mode 100644 vendor/github.com/slack-go/slack/usergroups.go delete mode 100644 vendor/github.com/slack-go/slack/users.go delete mode 100644 vendor/github.com/slack-go/slack/views.go delete mode 100644 vendor/github.com/slack-go/slack/webhooks.go delete mode 100644 vendor/github.com/slack-go/slack/websocket.go delete mode 100644 vendor/github.com/slack-go/slack/websocket_channels.go delete mode 100644 vendor/github.com/slack-go/slack/websocket_desktop_notification.go delete mode 100644 vendor/github.com/slack-go/slack/websocket_dm.go delete mode 100644 vendor/github.com/slack-go/slack/websocket_dnd.go delete mode 100644 vendor/github.com/slack-go/slack/websocket_files.go delete mode 100644 vendor/github.com/slack-go/slack/websocket_groups.go delete mode 100644 vendor/github.com/slack-go/slack/websocket_internals.go delete mode 100644 vendor/github.com/slack-go/slack/websocket_managed_conn.go delete mode 100644 vendor/github.com/slack-go/slack/websocket_misc.go delete mode 100644 vendor/github.com/slack-go/slack/websocket_mobile_in_app_notification.go delete mode 100644 vendor/github.com/slack-go/slack/websocket_pins.go delete mode 100644 vendor/github.com/slack-go/slack/websocket_reactions.go delete mode 100644 vendor/github.com/slack-go/slack/websocket_stars.go delete mode 100644 vendor/github.com/slack-go/slack/websocket_subteam.go delete mode 100644 vendor/github.com/slack-go/slack/websocket_teams.go delete mode 100644 vendor/github.com/slack-go/slack/workflow_step.go delete mode 100644 vendor/github.com/slack-go/slack/workflow_step_execute.go delete mode 100644 vendor/golang.org/x/net/html/atom/atom.go delete mode 100644 vendor/golang.org/x/net/html/atom/table.go delete mode 100644 vendor/golang.org/x/net/html/const.go delete mode 100644 vendor/golang.org/x/net/html/doc.go delete mode 100644 vendor/golang.org/x/net/html/doctype.go delete mode 100644 vendor/golang.org/x/net/html/entity.go delete mode 100644 vendor/golang.org/x/net/html/escape.go delete mode 100644 vendor/golang.org/x/net/html/foreign.go delete mode 100644 vendor/golang.org/x/net/html/node.go delete mode 100644 vendor/golang.org/x/net/html/parse.go delete mode 100644 vendor/golang.org/x/net/html/render.go delete mode 100644 vendor/golang.org/x/net/html/token.go delete mode 100644 vendor/golang.org/x/net/internal/socks/client.go delete mode 100644 vendor/golang.org/x/net/internal/socks/socks.go delete mode 100644 vendor/golang.org/x/net/proxy/dial.go delete mode 100644 vendor/golang.org/x/net/proxy/direct.go delete mode 100644 vendor/golang.org/x/net/proxy/per_host.go delete mode 100644 vendor/golang.org/x/net/proxy/proxy.go delete mode 100644 vendor/golang.org/x/net/proxy/socks5.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff98ff76..6c31a903 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,6 @@ jobs: run: | go test -v -timeout=100s -covermode=count -coverprofile=$GITHUB_WORKSPACE/profile.cov_tmp ./... cat $GITHUB_WORKSPACE/profile.cov_tmp | grep -v "mocks" | grep -v "_mock" > $GITHUB_WORKSPACE/profile.cov - working-directory: app env: GO111MODULE: on TZ: "America/Chicago" @@ -87,7 +86,7 @@ jobs: -t ${USERNAME}/tg-spam:${ref} -t ${USERNAME}/tg-spam:latest . - name: remote deployment from master - if: ${{ github.ref == 'refs/heads/master' }} + if: ${{ startsWith(github.ref, 'refs/tags/') }} env: UPDATER_KEY: ${{ secrets.UPDATER_KEY }} run: curl https://radio-t.com/updater/update/tg-spam/${UPDATER_KEY} diff --git a/Makefile b/Makefile index 3efac0bd..0620f2a5 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ docker: docker build -t umputun/tg-spam . race_test: - cd app && go test -race -mod=vendor -timeout=60s -count 1 ./... + go test -race -mod=vendor -timeout=60s -count 1 ./... prep_site: cp -fv README.md site/docs/index.md diff --git a/README.md b/README.md index bcf1835a..810242c6 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ To allow such a feature, some parameters in `admin` section must be specified: ### Updating spam samples dynamically -The bot can be configured to update spam samples dynamically. To enable this feature, reporting to the admin chat must be enabled (see `--admin.url=, [$ADMIN_URL]` above. If any of privileged users (`--super=, [$SUPER_USER]`) forward a message to admin chat, the bot will add this message to the internal spam samples file (`spam-dynamic.txt`) and reload it. This allows the bot to learn new spam patterns on the fly. +The bot can be configured to update spam samples dynamically. To enable this feature, reporting to the admin chat must be enabled (see `--admin.url=, [$ADMIN_URL]` above. If any of privileged users (`--super=, [$SUPER_USER]`) forwards a message to admin chat, the bot will add this message to the internal spam samples file (`spam-dynamic.txt`) and reload it. This allows the bot to learn new spam patterns on the fly. In addition, the bot will do the best to remove the original spam message from the group and ban the user who sent it. This is not always possible, as the forwarding strips the original user id. To address this limitation, tg-spam keeps the list of latest messages (in fact, it stores hashes) associated with the user id and the message id. This information is used to find the original message and ban the user. Note: if the bot is running in docker container, `--files.dynamic-spam=, [$FILES_DYNAMIC_SPAM]` must be set to the mapped volume's location to stay persistent after container restart. @@ -128,6 +128,7 @@ Use this token to access the HTTP API: ``` --testing-id= testing ids, allow bot to reply to them [$TESTING_ID] + --history-duration= history duration (default: 1h) [$HISTORY_DURATION] --super= super-users [$SUPER_USER] --no-spam-reply do not reply to spam messages [$NO_SPAM_REPLY] --similarity-threshold= spam threshold (default: 0.5) [$SIMILARITY_THRESHOLD] @@ -176,4 +177,42 @@ message: Help Options: -h, --help Show this help message -``` \ No newline at end of file +``` + +## Using tg-spam as a library + +The bot can be used as a library as well. To do so, import the `github.com/umputun/tg-spam/lib` package and create a new instance of the `Detector` struct. Then, call the `Check` method with the message and userID to check. The method will return `true` if the message is spam and `false` otherwise. In addition, the `Check` method will return the list of applied rules as well as the spam-related details. + +For more details see the [TBD]() + +Example: + +```go +package main + +import ( + "io" + + tgspam "github.com/umputun/tg-spam/lib" +) + +func main() { + detector := tgspam.NewDetector(tgspam.Config{ + SimilarityThreshold: 0.5, + MinMsgLen: 50, + MaxEmoji: 2, + FirstMessageOnly: false, + HTTPClient: &http.Client{Timeout: 30 * time.Second}, + }) + + // prepare samples and exclude tokens + spamSample := bytes.NewBufferString("this is spam\nwin a prize\n") // need io.Reader, in real life it will be a file + hamSample := bytes.NewBufferString("this is ham\n") + excludeTokens := bytes.NewBufferString(`"a", "the"`) + + // load samples + detector.LoadSamples(excludeTokens, []io.Reader{spamSample}, []io.Reader{hamSample}) + + isSpam, details := detector.Check("this is spam", 123456) +} +``` diff --git a/app/bot/bot.go b/app/bot/bot.go index 1db70b1c..5b23bb8c 100644 --- a/app/bot/bot.go +++ b/app/bot/bot.go @@ -9,6 +9,11 @@ import ( //go:generate moq --out mocks/http_client.go --pkg mocks --skip-ensure . HTTPClient:HTTPClient +// PermanentBanDuration defines duration of permanent ban: +// If user is restricted for more than 366 days or less than 30 seconds from the current time, +// they are considered to be restricted forever. +var PermanentBanDuration = time.Hour * 24 * 400 + // Response describes bot's reaction on particular message type Response struct { Text string diff --git a/app/bot/file_watcher.go b/app/bot/file_watcher.go deleted file mode 100644 index c6960e91..00000000 --- a/app/bot/file_watcher.go +++ /dev/null @@ -1,111 +0,0 @@ -package bot - -import ( - "bytes" - "context" - "fmt" - "io" - "log" - "os" - "sync" - - "github.com/fsnotify/fsnotify" -) - -// watch starts watching file for changes and calls onDataChange callback -// this is a helper for dynamic reloading of files used by SpamFilter -func watch(ctx context.Context, path string, onDataChange func(io.Reader) error) error { - watcher, err := fsnotify.NewWatcher() - if err != nil { - return fmt.Errorf("failed to create watcher: %w", err) - } - defer watcher.Close() - - done := make(chan bool) - go func() { - defer close(done) - for { - select { - case <-ctx.Done(): - log.Printf("[INFO] stopping watcher for %s, %v", path, ctx.Err()) - return - case event, ok := <-watcher.Events: - if !ok { - return - } - if event.Has(fsnotify.Write) || event.Has(fsnotify.Create) { - data, e := readFile(path) - if e != nil { - log.Printf("[WARN] failed to read updated file %s: %v", path, e) - continue - } - if e = onDataChange(data); e != nil { - log.Printf("[WARN] failed to load updated file %s: %v", path, e) - continue - } - } - case e, ok := <-watcher.Errors: - if !ok { - return - } - log.Printf("[WARN] watcher error: %v", e) - } - } - }() - - err = watcher.Add(path) - if err != nil { - return fmt.Errorf("failed to add %s to watcher: %w", path, err) - } - <-done - return nil -} - -// watchPair starts watching two files for changes and calls onDataChange callback -func watchPair(ctx context.Context, path1, path2 string, onDataChange func(io.Reader, io.Reader) error) { - var wg sync.WaitGroup - wg.Add(2) - - go func() { - defer wg.Done() - err := watch(ctx, path1, func(r io.Reader) error { - r2, err := readFile(path2) - if err != nil { - return err - } - return onDataChange(r, r2) - }) - if err != nil { - log.Printf("[WARN] failed to watch file %s: %v", path1, err) - } - }() - - go func() { - defer wg.Done() - err := watch(ctx, path2, func(r io.Reader) error { - r1, err := readFile(path1) - if err != nil { - return err - } - return onDataChange(r1, r) - }) - if err != nil { - log.Printf("[WARN] failed to watch file %s: %v", path2, err) - } - }() - - wg.Wait() -} - -func readFile(path string) (io.Reader, error) { - file, err := os.Open(path) //nolint gosec // path is controlled by the app - if err != nil { - return nil, fmt.Errorf("failed to open file %s: %w", path, err) - } - defer file.Close() - data, err := io.ReadAll(file) - if err != nil { - return nil, fmt.Errorf("failed to read file %s: %w", path, err) - } - return bytes.NewReader(data), nil -} diff --git a/app/bot/file_watcher_test.go b/app/bot/file_watcher_test.go deleted file mode 100644 index 568f6cdc..00000000 --- a/app/bot/file_watcher_test.go +++ /dev/null @@ -1,138 +0,0 @@ -package bot - -import ( - "context" - "io" - "os" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestWatch(t *testing.T) { - tmpfile, err := os.CreateTemp("", "watcher") - require.NoError(t, err) - defer os.Remove(tmpfile.Name()) // clean up - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - dataChangeCalled := false - var dataChangeContent string - onDataChange := func(r io.Reader) error { - dataChangeCalled = true - data, e := io.ReadAll(r) - require.NoError(t, e) - dataChangeContent = string(data) - return nil - } - - time.AfterFunc(time.Millisecond*500, func() { - _, err = tmpfile.WriteString("hello world") - require.NoError(t, err) - tmpfile.Close() - - time.Sleep(time.Millisecond * 100) // don't cancel too early, wait for onDataChange to be called - cancel() - }) - - err = watch(ctx, tmpfile.Name(), onDataChange) - assert.NoError(t, err) - assert.True(t, dataChangeCalled, "onDataChange should have been called") - assert.Equal(t, "hello world", dataChangeContent, "onDataChange should have received the correct data") -} - -func TestWatchPair_bothFilesChanged(t *testing.T) { - tmpfile1, err := os.CreateTemp("", "watcher1") - require.NoError(t, err) - defer os.Remove(tmpfile1.Name()) - - tmpfile2, err := os.CreateTemp("", "watcher2") - require.NoError(t, err) - defer os.Remove(tmpfile2.Name()) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - dataChangeCalled := false - var dataChangeContent1, dataChangeContent2 string - var lock sync.Mutex - onDataChange := func(r1, r2 io.Reader) error { - lock.Lock() - defer lock.Unlock() - t.Log("onDataChange called") - dataChangeCalled = true - data1, e := io.ReadAll(r1) - require.NoError(t, e) - dataChangeContent1 = string(data1) - - data2, e := io.ReadAll(r2) - require.NoError(t, e) - dataChangeContent2 = string(data2) - return nil - } - - time.AfterFunc(time.Millisecond*500, func() { - _, err = tmpfile1.WriteString("hello world 1") - require.NoError(t, err) - tmpfile1.Close() - - _, err = tmpfile2.WriteString("hello world 2") - require.NoError(t, err) - tmpfile2.Close() - - time.Sleep(time.Millisecond * 100) // don't cancel too early, wait for onDataChange to be called - cancel() - }) - - watchPair(ctx, tmpfile1.Name(), tmpfile2.Name(), onDataChange) - require.True(t, dataChangeCalled, "onDataChange should have been called") - assert.Equal(t, "hello world 1", dataChangeContent1, "onDataChange should have received the correct data from file 1") - assert.Equal(t, "hello world 2", dataChangeContent2, "onDataChange should have received the correct data from file 2") -} - -func TestWatchPair_oneFileChanged(t *testing.T) { - tmpfile1, err := os.CreateTemp("", "watcher1") - require.NoError(t, err) - defer os.Remove(tmpfile1.Name()) // clean up - - tmpfile2, err := os.CreateTemp("", "watcher2") - require.NoError(t, err) - defer os.Remove(tmpfile2.Name()) // clean up - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - dataChangeCalled := false - var dataChangeContent1, dataChangeContent2 string - onDataChange := func(r1, r2 io.Reader) error { - t.Log("onDataChange called") - dataChangeCalled = true - data1, e := io.ReadAll(r1) - require.NoError(t, e) - - dataChangeContent1 = string(data1) - data2, e := io.ReadAll(r2) - require.NoError(t, e) - - dataChangeContent2 = string(data2) - return nil - } - - time.AfterFunc(time.Millisecond*500, func() { - _, err = tmpfile1.WriteString("hello world 1") - require.NoError(t, err) - tmpfile1.Close() - // do not write to tmpfile2 - time.Sleep(time.Millisecond * 100) // don't cancel too early, wait for onDataChange to be called - cancel() - }) - - watchPair(ctx, tmpfile1.Name(), tmpfile2.Name(), onDataChange) - require.True(t, dataChangeCalled, "onDataChange should have been called") - assert.Equal(t, "hello world 1", dataChangeContent1, "onDataChange should have received the correct data from file 1") - assert.Equal(t, "", dataChangeContent2, "onDataChange should have received no data from file 2 because it was not changed") -} diff --git a/app/bot/mocks/detector.go b/app/bot/mocks/detector.go new file mode 100644 index 00000000..fa3d6a30 --- /dev/null +++ b/app/bot/mocks/detector.go @@ -0,0 +1,391 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package mocks + +import ( + "github.com/umputun/tg-spam/lib" + "io" + "sync" +) + +// DetectorMock is a mock implementation of bot.Detector. +// +// func TestSomethingThatUsesDetector(t *testing.T) { +// +// // make and configure a mocked bot.Detector +// mockedDetector := &DetectorMock{ +// CheckFunc: func(msg string, userID int64) (bool, []lib.CheckResult) { +// panic("mock out the Check method") +// }, +// LoadSamplesFunc: func(exclReader io.Reader, spamReaders []io.Reader, hamReaders []io.Reader) (lib.LoadResult, error) { +// panic("mock out the LoadSamples method") +// }, +// LoadStopWordsFunc: func(readers ...io.Reader) (lib.LoadResult, error) { +// panic("mock out the LoadStopWords method") +// }, +// ResetFunc: func() { +// panic("mock out the Reset method") +// }, +// UpdateHamFunc: func(msg string) error { +// panic("mock out the UpdateHam method") +// }, +// UpdateSpamFunc: func(msg string) error { +// panic("mock out the UpdateSpam method") +// }, +// WithHamUpdaterFunc: func(s lib.SampleUpdater) { +// panic("mock out the WithHamUpdater method") +// }, +// WithSpamUpdaterFunc: func(s lib.SampleUpdater) { +// panic("mock out the WithSpamUpdater method") +// }, +// } +// +// // use mockedDetector in code that requires bot.Detector +// // and then make assertions. +// +// } +type DetectorMock struct { + // CheckFunc mocks the Check method. + CheckFunc func(msg string, userID int64) (bool, []lib.CheckResult) + + // LoadSamplesFunc mocks the LoadSamples method. + LoadSamplesFunc func(exclReader io.Reader, spamReaders []io.Reader, hamReaders []io.Reader) (lib.LoadResult, error) + + // LoadStopWordsFunc mocks the LoadStopWords method. + LoadStopWordsFunc func(readers ...io.Reader) (lib.LoadResult, error) + + // ResetFunc mocks the Reset method. + ResetFunc func() + + // UpdateHamFunc mocks the UpdateHam method. + UpdateHamFunc func(msg string) error + + // UpdateSpamFunc mocks the UpdateSpam method. + UpdateSpamFunc func(msg string) error + + // WithHamUpdaterFunc mocks the WithHamUpdater method. + WithHamUpdaterFunc func(s lib.SampleUpdater) + + // WithSpamUpdaterFunc mocks the WithSpamUpdater method. + WithSpamUpdaterFunc func(s lib.SampleUpdater) + + // calls tracks calls to the methods. + calls struct { + // Check holds details about calls to the Check method. + Check []struct { + // Msg is the msg argument value. + Msg string + // UserID is the userID argument value. + UserID int64 + } + // LoadSamples holds details about calls to the LoadSamples method. + LoadSamples []struct { + // ExclReader is the exclReader argument value. + ExclReader io.Reader + // SpamReaders is the spamReaders argument value. + SpamReaders []io.Reader + // HamReaders is the hamReaders argument value. + HamReaders []io.Reader + } + // LoadStopWords holds details about calls to the LoadStopWords method. + LoadStopWords []struct { + // Readers is the readers argument value. + Readers []io.Reader + } + // Reset holds details about calls to the Reset method. + Reset []struct { + } + // UpdateHam holds details about calls to the UpdateHam method. + UpdateHam []struct { + // Msg is the msg argument value. + Msg string + } + // UpdateSpam holds details about calls to the UpdateSpam method. + UpdateSpam []struct { + // Msg is the msg argument value. + Msg string + } + // WithHamUpdater holds details about calls to the WithHamUpdater method. + WithHamUpdater []struct { + // S is the s argument value. + S lib.SampleUpdater + } + // WithSpamUpdater holds details about calls to the WithSpamUpdater method. + WithSpamUpdater []struct { + // S is the s argument value. + S lib.SampleUpdater + } + } + lockCheck sync.RWMutex + lockLoadSamples sync.RWMutex + lockLoadStopWords sync.RWMutex + lockReset sync.RWMutex + lockUpdateHam sync.RWMutex + lockUpdateSpam sync.RWMutex + lockWithHamUpdater sync.RWMutex + lockWithSpamUpdater sync.RWMutex +} + +// Check calls CheckFunc. +func (mock *DetectorMock) Check(msg string, userID int64) (bool, []lib.CheckResult) { + if mock.CheckFunc == nil { + panic("DetectorMock.CheckFunc: method is nil but Detector.Check was just called") + } + callInfo := struct { + Msg string + UserID int64 + }{ + Msg: msg, + UserID: userID, + } + mock.lockCheck.Lock() + mock.calls.Check = append(mock.calls.Check, callInfo) + mock.lockCheck.Unlock() + return mock.CheckFunc(msg, userID) +} + +// CheckCalls gets all the calls that were made to Check. +// Check the length with: +// +// len(mockedDetector.CheckCalls()) +func (mock *DetectorMock) CheckCalls() []struct { + Msg string + UserID int64 +} { + var calls []struct { + Msg string + UserID int64 + } + mock.lockCheck.RLock() + calls = mock.calls.Check + mock.lockCheck.RUnlock() + return calls +} + +// LoadSamples calls LoadSamplesFunc. +func (mock *DetectorMock) LoadSamples(exclReader io.Reader, spamReaders []io.Reader, hamReaders []io.Reader) (lib.LoadResult, error) { + if mock.LoadSamplesFunc == nil { + panic("DetectorMock.LoadSamplesFunc: method is nil but Detector.LoadSamples was just called") + } + callInfo := struct { + ExclReader io.Reader + SpamReaders []io.Reader + HamReaders []io.Reader + }{ + ExclReader: exclReader, + SpamReaders: spamReaders, + HamReaders: hamReaders, + } + mock.lockLoadSamples.Lock() + mock.calls.LoadSamples = append(mock.calls.LoadSamples, callInfo) + mock.lockLoadSamples.Unlock() + return mock.LoadSamplesFunc(exclReader, spamReaders, hamReaders) +} + +// LoadSamplesCalls gets all the calls that were made to LoadSamples. +// Check the length with: +// +// len(mockedDetector.LoadSamplesCalls()) +func (mock *DetectorMock) LoadSamplesCalls() []struct { + ExclReader io.Reader + SpamReaders []io.Reader + HamReaders []io.Reader +} { + var calls []struct { + ExclReader io.Reader + SpamReaders []io.Reader + HamReaders []io.Reader + } + mock.lockLoadSamples.RLock() + calls = mock.calls.LoadSamples + mock.lockLoadSamples.RUnlock() + return calls +} + +// LoadStopWords calls LoadStopWordsFunc. +func (mock *DetectorMock) LoadStopWords(readers ...io.Reader) (lib.LoadResult, error) { + if mock.LoadStopWordsFunc == nil { + panic("DetectorMock.LoadStopWordsFunc: method is nil but Detector.LoadStopWords was just called") + } + callInfo := struct { + Readers []io.Reader + }{ + Readers: readers, + } + mock.lockLoadStopWords.Lock() + mock.calls.LoadStopWords = append(mock.calls.LoadStopWords, callInfo) + mock.lockLoadStopWords.Unlock() + return mock.LoadStopWordsFunc(readers...) +} + +// LoadStopWordsCalls gets all the calls that were made to LoadStopWords. +// Check the length with: +// +// len(mockedDetector.LoadStopWordsCalls()) +func (mock *DetectorMock) LoadStopWordsCalls() []struct { + Readers []io.Reader +} { + var calls []struct { + Readers []io.Reader + } + mock.lockLoadStopWords.RLock() + calls = mock.calls.LoadStopWords + mock.lockLoadStopWords.RUnlock() + return calls +} + +// Reset calls ResetFunc. +func (mock *DetectorMock) Reset() { + if mock.ResetFunc == nil { + panic("DetectorMock.ResetFunc: method is nil but Detector.Reset was just called") + } + callInfo := struct { + }{} + mock.lockReset.Lock() + mock.calls.Reset = append(mock.calls.Reset, callInfo) + mock.lockReset.Unlock() + mock.ResetFunc() +} + +// ResetCalls gets all the calls that were made to Reset. +// Check the length with: +// +// len(mockedDetector.ResetCalls()) +func (mock *DetectorMock) ResetCalls() []struct { +} { + var calls []struct { + } + mock.lockReset.RLock() + calls = mock.calls.Reset + mock.lockReset.RUnlock() + return calls +} + +// UpdateHam calls UpdateHamFunc. +func (mock *DetectorMock) UpdateHam(msg string) error { + if mock.UpdateHamFunc == nil { + panic("DetectorMock.UpdateHamFunc: method is nil but Detector.UpdateHam was just called") + } + callInfo := struct { + Msg string + }{ + Msg: msg, + } + mock.lockUpdateHam.Lock() + mock.calls.UpdateHam = append(mock.calls.UpdateHam, callInfo) + mock.lockUpdateHam.Unlock() + return mock.UpdateHamFunc(msg) +} + +// UpdateHamCalls gets all the calls that were made to UpdateHam. +// Check the length with: +// +// len(mockedDetector.UpdateHamCalls()) +func (mock *DetectorMock) UpdateHamCalls() []struct { + Msg string +} { + var calls []struct { + Msg string + } + mock.lockUpdateHam.RLock() + calls = mock.calls.UpdateHam + mock.lockUpdateHam.RUnlock() + return calls +} + +// UpdateSpam calls UpdateSpamFunc. +func (mock *DetectorMock) UpdateSpam(msg string) error { + if mock.UpdateSpamFunc == nil { + panic("DetectorMock.UpdateSpamFunc: method is nil but Detector.UpdateSpam was just called") + } + callInfo := struct { + Msg string + }{ + Msg: msg, + } + mock.lockUpdateSpam.Lock() + mock.calls.UpdateSpam = append(mock.calls.UpdateSpam, callInfo) + mock.lockUpdateSpam.Unlock() + return mock.UpdateSpamFunc(msg) +} + +// UpdateSpamCalls gets all the calls that were made to UpdateSpam. +// Check the length with: +// +// len(mockedDetector.UpdateSpamCalls()) +func (mock *DetectorMock) UpdateSpamCalls() []struct { + Msg string +} { + var calls []struct { + Msg string + } + mock.lockUpdateSpam.RLock() + calls = mock.calls.UpdateSpam + mock.lockUpdateSpam.RUnlock() + return calls +} + +// WithHamUpdater calls WithHamUpdaterFunc. +func (mock *DetectorMock) WithHamUpdater(s lib.SampleUpdater) { + if mock.WithHamUpdaterFunc == nil { + panic("DetectorMock.WithHamUpdaterFunc: method is nil but Detector.WithHamUpdater was just called") + } + callInfo := struct { + S lib.SampleUpdater + }{ + S: s, + } + mock.lockWithHamUpdater.Lock() + mock.calls.WithHamUpdater = append(mock.calls.WithHamUpdater, callInfo) + mock.lockWithHamUpdater.Unlock() + mock.WithHamUpdaterFunc(s) +} + +// WithHamUpdaterCalls gets all the calls that were made to WithHamUpdater. +// Check the length with: +// +// len(mockedDetector.WithHamUpdaterCalls()) +func (mock *DetectorMock) WithHamUpdaterCalls() []struct { + S lib.SampleUpdater +} { + var calls []struct { + S lib.SampleUpdater + } + mock.lockWithHamUpdater.RLock() + calls = mock.calls.WithHamUpdater + mock.lockWithHamUpdater.RUnlock() + return calls +} + +// WithSpamUpdater calls WithSpamUpdaterFunc. +func (mock *DetectorMock) WithSpamUpdater(s lib.SampleUpdater) { + if mock.WithSpamUpdaterFunc == nil { + panic("DetectorMock.WithSpamUpdaterFunc: method is nil but Detector.WithSpamUpdater was just called") + } + callInfo := struct { + S lib.SampleUpdater + }{ + S: s, + } + mock.lockWithSpamUpdater.Lock() + mock.calls.WithSpamUpdater = append(mock.calls.WithSpamUpdater, callInfo) + mock.lockWithSpamUpdater.Unlock() + mock.WithSpamUpdaterFunc(s) +} + +// WithSpamUpdaterCalls gets all the calls that were made to WithSpamUpdater. +// Check the length with: +// +// len(mockedDetector.WithSpamUpdaterCalls()) +func (mock *DetectorMock) WithSpamUpdaterCalls() []struct { + S lib.SampleUpdater +} { + var calls []struct { + S lib.SampleUpdater + } + mock.lockWithSpamUpdater.RLock() + calls = mock.calls.WithSpamUpdater + mock.lockWithSpamUpdater.RUnlock() + return calls +} diff --git a/app/bot/sample_updater.go b/app/bot/sample_updater.go index 1eb23758..2657c1be 100644 --- a/app/bot/sample_updater.go +++ b/app/bot/sample_updater.go @@ -7,19 +7,19 @@ import ( "strings" ) -// sampleUpdater represents a file that can be read and appended to -// this is a helper for dynamic reloading of files used by SpamFilter -type sampleUpdater struct { +// SampleUpdater represents a file that can be read and appended to. +// this is a helper for dynamic reloading of samples used by SpamFilter +type SampleUpdater struct { fileName string } -// newSampleUpdater creates a new sampleUpdater -func newSampleUpdater(fileName string) *sampleUpdater { - return &sampleUpdater{fileName: fileName} +// NewSampleUpdater creates a new SampleUpdater +func NewSampleUpdater(fileName string) *SampleUpdater { + return &SampleUpdater{fileName: fileName} } // Reader returns a reader for the file, caller must close it -func (s *sampleUpdater) Reader() (io.ReadCloser, error) { +func (s *SampleUpdater) Reader() (io.ReadCloser, error) { fh, err := os.Open(s.fileName) if err != nil { return nil, fmt.Errorf("failed to open %s: %w", s.fileName, err) @@ -28,7 +28,7 @@ func (s *sampleUpdater) Reader() (io.ReadCloser, error) { } // Append a message to the file -func (s *sampleUpdater) Append(msg string) error { +func (s *SampleUpdater) Append(msg string) error { fh, err := os.OpenFile(s.fileName, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o644) //nolint:gosec // keep it readable by all if err != nil { return fmt.Errorf("failed to open %s: %w", s.fileName, err) diff --git a/app/bot/sample_updater_test.go b/app/bot/sample_updater_test.go index 50e3666c..38d58872 100644 --- a/app/bot/sample_updater_test.go +++ b/app/bot/sample_updater_test.go @@ -15,7 +15,7 @@ func TestSampleUpdater(t *testing.T) { require.NoError(t, err) defer os.Remove(file.Name()) - updater := newSampleUpdater(file.Name()) + updater := NewSampleUpdater(file.Name()) err = updater.Append("Test message") assert.NoError(t, err) @@ -33,7 +33,7 @@ func TestSampleUpdater(t *testing.T) { require.NoError(t, err) defer os.Remove(file.Name()) - updater := newSampleUpdater(file.Name()) + updater := NewSampleUpdater(file.Name()) err = updater.Append("Test message\nsecond line\nthird line") assert.NoError(t, err) @@ -47,7 +47,7 @@ func TestSampleUpdater(t *testing.T) { }) t.Run("unhappy path", func(t *testing.T) { - updater := newSampleUpdater("/tmp/non-existent/samples.txt") + updater := NewSampleUpdater("/tmp/non-existent/samples.txt") err := updater.Append("Test message") assert.Error(t, err) _, err = updater.Reader() diff --git a/app/bot/spam.go b/app/bot/spam.go index 3855273a..df39a8a4 100644 --- a/app/bot/spam.go +++ b/app/bot/spam.go @@ -1,502 +1,228 @@ package bot import ( - "bufio" "bytes" "context" - "encoding/json" "fmt" "io" "log" - "math" - "net/http" "os" - "regexp" - "strings" - "sync" "time" + + "github.com/fsnotify/fsnotify" + "github.com/hashicorp/go-multierror" + + "github.com/umputun/tg-spam/lib" ) //go:generate moq --out mocks/sample_updater.go --pkg mocks --skip-ensure . sampleUpdaterInterface:SampleUpdater +//go:generate moq --out mocks/detector.go --pkg mocks --skip-ensure . Detector -// SpamFilter bot checks if a user is a spammer using internal matching as well as CAS API. +// SpamFilter bot checks if a user is a spammer using lib.Detector // Reloads spam samples, stop words and excluded tokens on file change. -// Synchronized on mutex, only on public OnMessage method and load methods. -// Calling private methods directly (usually from tests) for spam check is unsafe. type SpamFilter struct { - SpamParams - - tokenizedSpam []map[string]int - approvedUsers map[int64]bool - stopWords []string - excludedTokens []string - spamClassifier Classifier - - spamSamplesUpd sampleUpdaterInterface - hamSamplesUpd sampleUpdaterInterface + director Detector + params SpamConfig - lock sync.RWMutex + // spamSamplesUpd sampleUpdaterInterface + // hamSamplesUpd sampleUpdaterInterface } -// If user is restricted for more than 366 days or less than 30 seconds from the current time, -// they are considered to be restricted forever. -var permanentBanDuration = time.Hour * 24 * 400 - -// SpamParams is a full set of parameters for spam bot -type SpamParams struct { - SimilarityThreshold float64 - MinMsgLen int - MaxAllowedEmoji int - CasAPI string - HTTPClient HTTPClient +// SpamConfig is a full set of parameters for spam bot +type SpamConfig struct { + // samples file names need to be watched for changes and reload. SpamSamplesFile string + HamSamplesFile string StopWordsFile string ExcludedTokensFile string SpamDynamicFile string HamDynamicFile string - HamSamplesFile string - ParanoidMode bool SpamMsg string SpamDryMsg string + WatchDelay time.Duration + Dry bool } -type sampleUpdaterInterface interface { - Append(msg string) error - Reader() (io.ReadCloser, error) +// Detector is a spam detector interface +type Detector interface { + Check(msg string, userID int64) (spam bool, cr []lib.CheckResult) + LoadSamples(exclReader io.Reader, spamReaders, hamReaders []io.Reader) (lib.LoadResult, error) + LoadStopWords(readers ...io.Reader) (lib.LoadResult, error) + UpdateSpam(msg string) error + UpdateHam(msg string) error } -// NewSpamFilter makes a spam detecting bot -func NewSpamFilter(ctx context.Context, p SpamParams) (*SpamFilter, error) { - log.Printf("[INFO] spam bot with %+v", p) - - init := func(s *SpamFilter) error { - tbl := []struct { - desc string - fileName string - loadFunc func(io.Reader) error - }{ - {desc: "excluded tokens", fileName: s.ExcludedTokensFile, loadFunc: s.loadExcludedTokens}, - {desc: "spam samples", fileName: s.SpamSamplesFile, loadFunc: s.loadSpamSamples}, - {desc: "stop words", fileName: s.StopWordsFile, loadFunc: s.loadStopWords}, - } - - for _, t := range tbl { - fh, err := os.Open(t.fileName) - if err != nil { - return fmt.Errorf("can't open %s %s: %w", t.desc, t.fileName, err) - } - defer fh.Close() //nolint gosec - - // initial data load - if err := t.loadFunc(fh); err != nil { - return fmt.Errorf("can't load %s %s: %w", t.desc, t.fileName, err) - } - - fname, loadFunc, desc := t.fileName, t.loadFunc, t.desc - go func() { - if err := watch(ctx, fname, loadFunc); err != nil { - log.Printf("[WARN] failed to watch %s file %s, error=%v", desc, fname, err) - } - }() - } - - // load classifiers - fhSpam, err := os.Open(s.SpamSamplesFile) - if err != nil { - return fmt.Errorf("can't open spam samples %s: %w", s.SpamSamplesFile, err) - } - defer fhSpam.Close() //nolint gosec - - fhHam, err := os.Open(s.HamSamplesFile) - if err != nil { - return fmt.Errorf("can't open ham samples %s: %w", s.HamSamplesFile, err) - } - defer fhHam.Close() //nolint gosec - - loadClassifiers := func(fhSpam, fhHam io.Reader) error { - s.spamClassifier.Reset() - if err := s.loadSpamClassifier(fhSpam); err != nil { - return err - } - if err := s.loadHamClassifier(fhHam); err != nil { - return err - } - return nil - } - if err := loadClassifiers(fhSpam, fhHam); err != nil { - return fmt.Errorf("can't load classifiers: %w", err) +// NewSpamFilter creates new spam filter +func NewSpamFilter(ctx context.Context, detector Detector, params SpamConfig) *SpamFilter { + res := &SpamFilter{director: detector, params: params} + go func() { + if err := res.watch(ctx, params.WatchDelay); err != nil { + log.Printf("[WARN] samples file watcher failed: %v", err) } - - go watchPair(ctx, s.SpamSamplesFile, s.HamSamplesFile, loadClassifiers) - return nil - } - - res := &SpamFilter{ - SpamParams: p, - approvedUsers: map[int64]bool{}, - spamSamplesUpd: newSampleUpdater(p.SpamDynamicFile), - hamSamplesUpd: newSampleUpdater(p.HamDynamicFile), - } - if err := init(res); err != nil { - return nil, fmt.Errorf("failed to init spam filter: %w", err) - } - - if err := res.loadDynFiles(); err != nil { - return nil, fmt.Errorf("can't load dynamic data: %w", err) - } - - return res, nil + }() + return res } // OnMessage checks if user already approved and if not checks if user is a spammer func (s *SpamFilter) OnMessage(msg Message) (response Response) { - s.lock.RLock() - defer s.lock.RUnlock() - - if (s.approvedUsers[msg.From.ID] && !s.ParanoidMode) || msg.From.ID == 0 || len(msg.Text) < s.MinMsgLen { - return Response{} - } - displayUsername := DisplayName(msg) - - isEmojiSpam, _ := s.tooManyEmojis(msg.Text, s.MaxAllowedEmoji) - stopWordsSpam := s.hasStopWords(msg.Text) - similaritySpam := s.isSpamSimilarityHigh(msg.Text) - classifiedSpam := s.isSpamClassified(msg.Text) - - if similaritySpam || isEmojiSpam || stopWordsSpam || s.isCasSpam(msg.From.ID) || classifiedSpam { - log.Printf("[INFO] user %s detected as spammer, msg: %q", displayUsername, msg.Text) - msgPrefix := s.SpamMsg - if s.Dry { - msgPrefix = s.SpamDryMsg + isSpam, checkResults := s.director.Check(msg.Text, msg.From.ID) + if isSpam { + log.Printf("[INFO] user %s detected as spammer, msg: %q, %+v", displayUsername, msg.Text, checkResults) + msgPrefix := s.params.SpamMsg + if s.params.Dry { + msgPrefix = s.params.SpamDryMsg } spamRespMsg := fmt.Sprintf("%s: %q (%d)", msgPrefix, displayUsername, msg.From.ID) - return Response{Text: spamRespMsg, Send: true, ReplyTo: msg.ID, BanInterval: permanentBanDuration, + return Response{Text: spamRespMsg, Send: true, ReplyTo: msg.ID, BanInterval: PermanentBanDuration, DeleteReplyTo: true, User: User{Username: msg.From.Username, ID: msg.From.ID, DisplayName: msg.From.DisplayName}, } } - - if id := msg.From.ID; id != 0 { - s.approvedUsers[id] = true - log.Printf("[INFO] user %s is not a spammer id %d, added to aproved", displayUsername, msg.From.ID) - } return Response{} // not a spam } // UpdateSpam appends a message to the spam samples file and updates the classifier func (s *SpamFilter) UpdateSpam(msg string) error { log.Printf("[DEBUG] update spam samples with %q", msg) - if err := s.spamSamplesUpd.Append(msg); err != nil { + if err := s.director.UpdateSpam(msg); err != nil { return fmt.Errorf("can't update spam samples: %w", err) } - - // add to samples - tokenizedSpam := s.tokenize(msg) - s.tokenizedSpam = append(s.tokenizedSpam, tokenizedSpam) - - // add to classifier - msgRdr := bytes.NewBufferString(msg) - if err := s.loadSpamClassifier(msgRdr); err != nil { - log.Printf("[WARN] failed to add spam to classifier %q: %v", msg, err) - } return nil } // UpdateHam appends a message to the ham samples file and updates the classifier func (s *SpamFilter) UpdateHam(msg string) error { log.Printf("[DEBUG] update ham samples with %q", msg) - if err := s.hamSamplesUpd.Append(msg); err != nil { + if err := s.director.UpdateHam(msg); err != nil { return fmt.Errorf("can't update ham samples: %w", err) } - msgRdr := bytes.NewBufferString(msg) - if err := s.loadHamClassifier(msgRdr); err != nil { - log.Printf("[WARN] failed to add ham sample %q: %v", msg, err) - } return nil } -func (s *SpamFilter) loadSpamSamples(reader io.Reader) error { - s.lock.Lock() - defer s.lock.Unlock() - log.Printf("[DEBUG] refreshing spam samples") - s.tokenizedSpam = nil - for t := range tokenChan(reader) { - tokenizedSpam := s.tokenize(t) - s.tokenizedSpam = append(s.tokenizedSpam, tokenizedSpam) - } - log.Printf("[INFO] loaded %d spam samples", len(s.tokenizedSpam)) - return nil -} - -func (s *SpamFilter) loadStopWords(reader io.Reader) error { - s.lock.Lock() - defer s.lock.Unlock() - log.Printf("[DEBUG] refreshing stop words") - s.stopWords = []string{} - for t := range tokenChan(reader) { - s.stopWords = append(s.stopWords, strings.ToLower(t)) - } - log.Printf("[INFO] loaded %d stop words", len(s.stopWords)) - return nil -} - -func (s *SpamFilter) loadExcludedTokens(reader io.Reader) error { - s.lock.Lock() - defer s.lock.Unlock() - log.Printf("[DEBUG] refreshing excluded tokens") - s.excludedTokens = []string{} - for t := range tokenChan(reader) { - s.excludedTokens = append(s.excludedTokens, strings.ToLower(t)) +// watch watches for changes in samples files and reloads them +// delay is a time to wait after the last change before reloading to avoid multiple reloads +func (s *SpamFilter) watch(ctx context.Context, delay time.Duration) error { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return fmt.Errorf("failed to create watcher: %w", err) } - log.Printf("[INFO] loaded %d excluded tokens", len(s.excludedTokens)) - return nil -} - -func (s *SpamFilter) loadSpamClassifier(reader io.Reader) error { - s.lock.Lock() - defer s.lock.Unlock() - return s.loadClassifier(reader, "spam") -} - -func (s *SpamFilter) loadHamClassifier(reader io.Reader) error { - s.lock.Lock() - defer s.lock.Unlock() - return s.loadClassifier(reader, "ham") -} + defer watcher.Close() -func (s *SpamFilter) loadClassifier(reader io.Reader, class Class) error { - log.Printf("[DEBUG] refreshing classifier %s", class) - docs := []Document{} - for t := range tokenChan(reader) { // each line is a string with space separated tokens - tokenizedSpam := s.tokenize(t) - tokens := make([]string, 0, len(tokenizedSpam)) - for token := range tokenizedSpam { - tokens = append(tokens, token) - } - docs = append(docs, Document{Class: class, Tokens: tokens}) - } - s.spamClassifier.Learn(docs...) - log.Printf("[INFO] loaded %d %s samples", len(docs), class) - return nil -} + done := make(chan bool) + reloadTimer := time.NewTimer(delay) + reloadPending := false -func (s *SpamFilter) loadDynFiles() error { - log.Printf("[DEBUG] load dynamic data, spam: %s, ham: %s", s.SpamDynamicFile, s.HamDynamicFile) - // load spam dynamic to classifier, appends - if rdr, err := s.spamSamplesUpd.Reader(); err == nil { - defer rdr.Close() //nolint gosec - if err := s.loadClassifier(rdr, "spam"); err != nil { - return fmt.Errorf("can't load dynamic spam classifier: %w", err) + go func() { + defer close(done) + for { + select { + case <-ctx.Done(): + log.Printf("[INFO] stopping watcher for samples: %v", ctx.Err()) + return + case event, ok := <-watcher.Events: + if !ok { + return + } + if event.Has(fsnotify.Write) || event.Has(fsnotify.Create) { + log.Printf("[DEBUG] file %q updated, op: %v", event.Name, event.Op) + if !reloadPending { + reloadPending = true + reloadTimer.Reset(delay) + } + } + case <-reloadTimer.C: + if reloadPending { + reloadPending = false + if err := s.ReloadSamples(); err != nil { + log.Printf("[WARN] %v", err) + } + } + case e, ok := <-watcher.Errors: + if !ok { + return + } + log.Printf("[WARN] watcher error: %v", e) + } } - } + }() - // load spam dynamic to samples - if rdr, err := s.spamSamplesUpd.Reader(); err == nil { - defer rdr.Close() //nolint gosec - for t := range tokenChan(rdr) { - // append tokens to tokenizedSpam - tokenizedSpam := s.tokenize(t) - s.tokenizedSpam = append(s.tokenizedSpam, tokenizedSpam) + errs := new(multierror.Error) + addToWatcher := func(file string) error { + if _, err := os.Stat(file); err != nil { + return fmt.Errorf("failed to stat file %q: %w", file, err) } + log.Printf("[DEBUG] add file %q to watcher", file) + return watcher.Add(file) } - - // load ham dynamic to classifier, appends - if rdr, err := s.hamSamplesUpd.Reader(); err == nil { - defer rdr.Close() //nolint gosec - if err := s.loadClassifier(rdr, "ham"); err != nil { - return fmt.Errorf("can't load dynamic ham classifier: %w", err) - } + errs = multierror.Append(errs, addToWatcher(s.params.ExcludedTokensFile)) + errs = multierror.Append(errs, addToWatcher(s.params.SpamSamplesFile)) + errs = multierror.Append(errs, addToWatcher(s.params.HamSamplesFile)) + errs = multierror.Append(errs, addToWatcher(s.params.StopWordsFile)) + if err := errs.ErrorOrNil(); err != nil { + return fmt.Errorf("failed to add some files to watcher: %w", err) } - + <-done return nil } -// isSpam checks if a given message is similar to any of the known bad messages. -func (s *SpamFilter) isSpamSimilarityHigh(message string) bool { - // check for spam similarity - tokenizedMessage := s.tokenize(message) - maxSimilarity, maxTokens, maxID := 0.0, map[string]int{}, 0 - for i, spam := range s.tokenizedSpam { - similarity := s.cosineSimilarity(tokenizedMessage, spam) - if similarity > maxSimilarity { - maxSimilarity = similarity - maxTokens = spam - maxID = i + 1 - } - if similarity >= s.SimilarityThreshold { - log.Printf("[DEBUG] high spam similarity: %0.2f, line %d, toekns: %v", - maxSimilarity, maxID, maxTokens) - return true - } - } - log.Printf("[DEBUG] low spam similarity: %0.2f", maxSimilarity) - return false -} +// ReloadSamples reloads samples and stop-words +func (s *SpamFilter) ReloadSamples() (err error) { + log.Printf("[DEBUG] reloading samples") -func (s *SpamFilter) isCasSpam(msgID int64) bool { - reqURL := fmt.Sprintf("%s/check?user_id=%d", s.CasAPI, msgID) - req, err := http.NewRequest("GET", reqURL, http.NoBody) - if err != nil { - log.Printf("[WARN] failed to make request %s, error=%v", reqURL, err) - return false - } - - resp, err := s.HTTPClient.Do(req) - if err != nil { - log.Printf("[WARN] failed to send request %s, error=%v", reqURL, err) - return false - } - defer resp.Body.Close() - - respData := struct { - OK bool `json:"ok"` // ok means user is a spammer - Description string `json:"description"` - }{} - - if err := json.NewDecoder(resp.Body).Decode(&respData); err != nil { - log.Printf("[WARN] failed to parse response from %s, error=%v", reqURL, err) - return false - } - if respData.OK { - log.Printf("[INFO] user %q detected as spammer: %s", msgID, respData.Description) - } - return respData.OK -} + var exclReader, spamReader, hamReader, stopWordsReader, spamDynamicReader, hamDynamicReader io.ReadCloser -func (s *SpamFilter) isSpamClassified(message string) bool { - // Classify tokens from a document - tm := s.tokenize(message) - tokens := make([]string, 0, len(tm)) - for token := range tm { - tokens = append(tokens, token) + // open mandatory spam and ham samples files + if spamReader, err = os.Open(s.params.SpamSamplesFile); err != nil { + return fmt.Errorf("failed to open spam samples file %q: %w", s.params.SpamSamplesFile, err) } - allScores, class, certain := s.spamClassifier.Classify(tokens...) - log.Printf("[DEBUG] spam classifier: %v, %s, %v", allScores, class, certain) - return class == "spam" && certain -} + defer spamReader.Close() -// tokenize takes a string and returns a map where the keys are unique words (tokens) -// and the values are the frequencies of those words in the string. -// exclude tokens representing common words. -func (s *SpamFilter) tokenize(inp string) map[string]int { - isExcludedToken := func(token string) bool { - for _, w := range s.excludedTokens { - if strings.EqualFold(token, w) { - return true - } - } - return false + if hamReader, err = os.Open(s.params.HamSamplesFile); err != nil { + return fmt.Errorf("failed to open ham samples file %q: %w", s.params.HamSamplesFile, err) } + defer hamReader.Close() - tokenFrequency := make(map[string]int) - tokens := strings.Fields(inp) - for _, token := range tokens { - if isExcludedToken(token) { - continue - } - token = s.cleanEmoji(token) - token = strings.Trim(token, ".,!?-:;()#") - token = strings.ToLower(token) - if len([]rune(token)) < 3 { - continue - } - tokenFrequency[strings.ToLower(token)]++ + // stop-words are optional + if stopWordsReader, err = os.Open(s.params.StopWordsFile); err != nil { + stopWordsReader = io.NopCloser(bytes.NewReader([]byte(""))) } - return tokenFrequency -} + defer stopWordsReader.Close() -// cosineSimilarity calculates the cosine similarity between two token frequency maps. -func (s *SpamFilter) cosineSimilarity(a, b map[string]int) float64 { - if len(a) == 0 || len(b) == 0 { - return 0.0 + // excluded tokens are optional + if exclReader, err = os.Open(s.params.ExcludedTokensFile); err != nil { + exclReader = io.NopCloser(bytes.NewReader([]byte(""))) } + defer exclReader.Close() - dotProduct := 0 // sum of product of corresponding frequencies - normA, normB := 0, 0 // square root of sum of squares of frequencies - - for key, val := range a { - dotProduct += val * b[key] - normA += val * val - } - for _, val := range b { - normB += val * val + // dynamic samples are optional + if spamDynamicReader, err = os.Open(s.params.SpamDynamicFile); err != nil { + spamDynamicReader = io.NopCloser(bytes.NewReader([]byte(""))) } + defer spamDynamicReader.Close() - if normA == 0 || normB == 0 { - return 0.0 + if hamDynamicReader, err = os.Open(s.params.HamDynamicFile); err != nil { + hamDynamicReader = io.NopCloser(bytes.NewReader([]byte(""))) } + defer hamDynamicReader.Close() - // cosine similarity formula - return float64(dotProduct) / (math.Sqrt(float64(normA)) * math.Sqrt(float64(normB))) -} - -func (s *SpamFilter) hasStopWords(message string) bool { - s.lock.RLock() - defer s.lock.RUnlock() - lowerCaseMessage := strings.ToLower(message) - lowerCaseMessage = emojiPattern.ReplaceAllString(lowerCaseMessage, "") - for _, word := range s.stopWords { - if strings.Contains(lowerCaseMessage, strings.ToLower(word)) { - log.Printf("[DEBUG] spam stop word %q", word) - return true - } + // reload samples and stop-words. note: we don't need Reset as LoadSamples and LoadStopWords clear the state first + lr, err := s.director.LoadSamples(exclReader, []io.Reader{spamReader, spamDynamicReader}, + []io.Reader{hamReader, hamDynamicReader}) + if err != nil { + return fmt.Errorf("failed to reload samples: %w", err) } - return false -} -func (s *SpamFilter) tooManyEmojis(message string, threshold int) (ok bool, count int) { - matches := emojiPattern.FindAllString(message, -1) - if len(matches) > threshold { - log.Printf("[DEBUG] spam emojis, %d of %d", len(matches), threshold) - return true, len(matches) + ls, err := s.director.LoadStopWords(stopWordsReader) + if err != nil { + return fmt.Errorf("failed to reload stop words: %w", err) } - return false, len(matches) -} - -func (s *SpamFilter) cleanEmoji(message string) string { - return emojiPattern.ReplaceAllString(message, "") -} -// tokenChan parses a file and returns a channel of tokens. -// A line per-token or comma-separated "tokens" supported -func tokenChan(reader io.Reader) <-chan string { - resCh := make(chan string) + log.Printf("[INFO] loaded samples - spam: %d, ham: %d, excluded tokens: %d, stop-words: %d", + lr.SpamSamples, lr.HamSamples, lr.ExcludedTokens, ls.StopWords) - go func() { - defer close(resCh) - - scanner := bufio.NewScanner(reader) - for scanner.Scan() { - line := scanner.Text() - if strings.Contains(line, ",") && strings.HasPrefix(line, "\"") { - // line with comma-separated tokens - lineTokens := strings.Split(line, ",") - for _, token := range lineTokens { - cleanToken := strings.Trim(token, " \"\n\r\t") - if cleanToken != "" { - resCh <- cleanToken - } - } - continue - } - // each line with a single token - cleanToken := strings.Trim(line, " \n\r\t") - if cleanToken != "" { - resCh <- cleanToken - } - } - - if err := scanner.Err(); err != nil { - log.Printf("[WARN] failed to read tokens, error=%v", err) - } - }() - - return resCh + return nil } - -// borrowed from https://stackoverflow.com/a/72255061 -var emojiPattern = regexp.MustCompile(`[#*0-9]\x{FE0F}?\x{20E3}|©\x{FE0F}?|[®\x{203C}\x{2049}\x{2122}\x{2139}\x{2194}-\x{2199}\x{21A9}\x{21AA}]\x{FE0F}?|[\x{231A}\x{231B}]|[\x{2328}\x{23CF}]\x{FE0F}?|[\x{23E9}-\x{23EC}]|[\x{23ED}-\x{23EF}]\x{FE0F}?|\x{23F0}|[\x{23F1}\x{23F2}]\x{FE0F}?|\x{23F3}|[\x{23F8}-\x{23FA}\x{24C2}\x{25AA}\x{25AB}\x{25B6}\x{25C0}\x{25FB}\x{25FC}]\x{FE0F}?|[\x{25FD}\x{25FE}]|[\x{2600}-\x{2604}\x{260E}\x{2611}]\x{FE0F}?|[\x{2614}\x{2615}]|\x{2618}\x{FE0F}?|\x{261D}[\x{FE0F}\x{1F3FB}-\x{1F3FF}]?|[\x{2620}\x{2622}\x{2623}\x{2626}\x{262A}\x{262E}\x{262F}\x{2638}-\x{263A}\x{2640}\x{2642}]\x{FE0F}?|[\x{2648}-\x{2653}]|[\x{265F}\x{2660}\x{2663}\x{2665}\x{2666}\x{2668}\x{267B}\x{267E}]\x{FE0F}?|\x{267F}|\x{2692}\x{FE0F}?|\x{2693}|[\x{2694}-\x{2697}\x{2699}\x{269B}\x{269C}\x{26A0}]\x{FE0F}?|\x{26A1}|\x{26A7}\x{FE0F}?|[\x{26AA}\x{26AB}]|[\x{26B0}\x{26B1}]\x{FE0F}?|[\x{26BD}\x{26BE}\x{26C4}\x{26C5}]|\x{26C8}\x{FE0F}?|\x{26CE}|[\x{26CF}\x{26D1}\x{26D3}]\x{FE0F}?|\x{26D4}|\x{26E9}\x{FE0F}?|\x{26EA}|[\x{26F0}\x{26F1}]\x{FE0F}?|[\x{26F2}\x{26F3}]|\x{26F4}\x{FE0F}?|\x{26F5}|[\x{26F7}\x{26F8}]\x{FE0F}?|\x{26F9}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{FE0F}\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{26FA}\x{26FD}]|\x{2702}\x{FE0F}?|\x{2705}|[\x{2708}\x{2709}]\x{FE0F}?|[\x{270A}\x{270B}][\x{1F3FB}-\x{1F3FF}]?|[\x{270C}\x{270D}][\x{FE0F}\x{1F3FB}-\x{1F3FF}]?|\x{270F}\x{FE0F}?|[\x{2712}\x{2714}\x{2716}\x{271D}\x{2721}]\x{FE0F}?|\x{2728}|[\x{2733}\x{2734}\x{2744}\x{2747}]\x{FE0F}?|[\x{274C}\x{274E}\x{2753}-\x{2755}\x{2757}]|\x{2763}\x{FE0F}?|\x{2764}(?:\x{200D}[\x{1F525}\x{1FA79}]|\x{FE0F}(?:\x{200D}[\x{1F525}\x{1FA79}])?)?|[\x{2795}-\x{2797}]|\x{27A1}\x{FE0F}?|[\x{27B0}\x{27BF}]|[\x{2934}\x{2935}\x{2B05}-\x{2B07}]\x{FE0F}?|[\x{2B1B}\x{2B1C}\x{2B50}\x{2B55}]|[\x{3030}\x{303D}\x{3297}\x{3299}]\x{FE0F}?|[\x{1F004}\x{1F0CF}]|[\x{1F170}\x{1F171}\x{1F17E}\x{1F17F}]\x{FE0F}?|[\x{1F18E}\x{1F191}-\x{1F19A}]|\x{1F1E6}[\x{1F1E8}-\x{1F1EC}\x{1F1EE}\x{1F1F1}\x{1F1F2}\x{1F1F4}\x{1F1F6}-\x{1F1FA}\x{1F1FC}\x{1F1FD}\x{1F1FF}]|\x{1F1E7}[\x{1F1E6}\x{1F1E7}\x{1F1E9}-\x{1F1EF}\x{1F1F1}-\x{1F1F4}\x{1F1F6}-\x{1F1F9}\x{1F1FB}\x{1F1FC}\x{1F1FE}\x{1F1FF}]|\x{1F1E8}[\x{1F1E6}\x{1F1E8}\x{1F1E9}\x{1F1EB}-\x{1F1EE}\x{1F1F0}-\x{1F1F5}\x{1F1F7}\x{1F1FA}-\x{1F1FF}]|\x{1F1E9}[\x{1F1EA}\x{1F1EC}\x{1F1EF}\x{1F1F0}\x{1F1F2}\x{1F1F4}\x{1F1FF}]|\x{1F1EA}[\x{1F1E6}\x{1F1E8}\x{1F1EA}\x{1F1EC}\x{1F1ED}\x{1F1F7}-\x{1F1FA}]|\x{1F1EB}[\x{1F1EE}-\x{1F1F0}\x{1F1F2}\x{1F1F4}\x{1F1F7}]|\x{1F1EC}[\x{1F1E6}\x{1F1E7}\x{1F1E9}-\x{1F1EE}\x{1F1F1}-\x{1F1F3}\x{1F1F5}-\x{1F1FA}\x{1F1FC}\x{1F1FE}]|\x{1F1ED}[\x{1F1F0}\x{1F1F2}\x{1F1F3}\x{1F1F7}\x{1F1F9}\x{1F1FA}]|\x{1F1EE}[\x{1F1E8}-\x{1F1EA}\x{1F1F1}-\x{1F1F4}\x{1F1F6}-\x{1F1F9}]|\x{1F1EF}[\x{1F1EA}\x{1F1F2}\x{1F1F4}\x{1F1F5}]|\x{1F1F0}[\x{1F1EA}\x{1F1EC}-\x{1F1EE}\x{1F1F2}\x{1F1F3}\x{1F1F5}\x{1F1F7}\x{1F1FC}\x{1F1FE}\x{1F1FF}]|\x{1F1F1}[\x{1F1E6}-\x{1F1E8}\x{1F1EE}\x{1F1F0}\x{1F1F7}-\x{1F1FB}\x{1F1FE}]|\x{1F1F2}[\x{1F1E6}\x{1F1E8}-\x{1F1ED}\x{1F1F0}-\x{1F1FF}]|\x{1F1F3}[\x{1F1E6}\x{1F1E8}\x{1F1EA}-\x{1F1EC}\x{1F1EE}\x{1F1F1}\x{1F1F4}\x{1F1F5}\x{1F1F7}\x{1F1FA}\x{1F1FF}]|\x{1F1F4}\x{1F1F2}|\x{1F1F5}[\x{1F1E6}\x{1F1EA}-\x{1F1ED}\x{1F1F0}-\x{1F1F3}\x{1F1F7}-\x{1F1F9}\x{1F1FC}\x{1F1FE}]|\x{1F1F6}\x{1F1E6}|\x{1F1F7}[\x{1F1EA}\x{1F1F4}\x{1F1F8}\x{1F1FA}\x{1F1FC}]|\x{1F1F8}[\x{1F1E6}-\x{1F1EA}\x{1F1EC}-\x{1F1F4}\x{1F1F7}-\x{1F1F9}\x{1F1FB}\x{1F1FD}-\x{1F1FF}]|\x{1F1F9}[\x{1F1E6}\x{1F1E8}\x{1F1E9}\x{1F1EB}-\x{1F1ED}\x{1F1EF}-\x{1F1F4}\x{1F1F7}\x{1F1F9}\x{1F1FB}\x{1F1FC}\x{1F1FF}]|\x{1F1FA}[\x{1F1E6}\x{1F1EC}\x{1F1F2}\x{1F1F3}\x{1F1F8}\x{1F1FE}\x{1F1FF}]|\x{1F1FB}[\x{1F1E6}\x{1F1E8}\x{1F1EA}\x{1F1EC}\x{1F1EE}\x{1F1F3}\x{1F1FA}]|\x{1F1FC}[\x{1F1EB}\x{1F1F8}]|\x{1F1FD}\x{1F1F0}|\x{1F1FE}[\x{1F1EA}\x{1F1F9}]|\x{1F1FF}[\x{1F1E6}\x{1F1F2}\x{1F1FC}]|\x{1F201}|\x{1F202}\x{FE0F}?|[\x{1F21A}\x{1F22F}\x{1F232}-\x{1F236}]|\x{1F237}\x{FE0F}?|[\x{1F238}-\x{1F23A}\x{1F250}\x{1F251}\x{1F300}-\x{1F320}]|[\x{1F321}\x{1F324}-\x{1F32C}]\x{FE0F}?|[\x{1F32D}-\x{1F335}]|\x{1F336}\x{FE0F}?|[\x{1F337}-\x{1F37C}]|\x{1F37D}\x{FE0F}?|[\x{1F37E}-\x{1F384}]|\x{1F385}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F386}-\x{1F393}]|[\x{1F396}\x{1F397}\x{1F399}-\x{1F39B}\x{1F39E}\x{1F39F}]\x{FE0F}?|[\x{1F3A0}-\x{1F3C1}]|\x{1F3C2}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F3C3}\x{1F3C4}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F3C5}\x{1F3C6}]|\x{1F3C7}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F3C8}\x{1F3C9}]|\x{1F3CA}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F3CB}\x{1F3CC}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{FE0F}\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F3CD}\x{1F3CE}]\x{FE0F}?|[\x{1F3CF}-\x{1F3D3}]|[\x{1F3D4}-\x{1F3DF}]\x{FE0F}?|[\x{1F3E0}-\x{1F3F0}]|\x{1F3F3}(?:\x{200D}(?:\x{26A7}\x{FE0F}?|\x{1F308})|\x{FE0F}(?:\x{200D}(?:\x{26A7}\x{FE0F}?|\x{1F308}))?)?|\x{1F3F4}(?:\x{200D}\x{2620}\x{FE0F}?|\x{E0067}\x{E0062}(?:\x{E0065}\x{E006E}\x{E0067}|\x{E0073}\x{E0063}\x{E0074}|\x{E0077}\x{E006C}\x{E0073})\x{E007F})?|[\x{1F3F5}\x{1F3F7}]\x{FE0F}?|[\x{1F3F8}-\x{1F407}]|\x{1F408}(?:\x{200D}\x{2B1B})?|[\x{1F409}-\x{1F414}]|\x{1F415}(?:\x{200D}\x{1F9BA})?|[\x{1F416}-\x{1F43A}]|\x{1F43B}(?:\x{200D}\x{2744}\x{FE0F}?)?|[\x{1F43C}-\x{1F43E}]|\x{1F43F}\x{FE0F}?|\x{1F440}|\x{1F441}(?:\x{200D}\x{1F5E8}\x{FE0F}?|\x{FE0F}(?:\x{200D}\x{1F5E8}\x{FE0F}?)?)?|[\x{1F442}\x{1F443}][\x{1F3FB}-\x{1F3FF}]?|[\x{1F444}\x{1F445}]|[\x{1F446}-\x{1F450}][\x{1F3FB}-\x{1F3FF}]?|[\x{1F451}-\x{1F465}]|[\x{1F466}\x{1F467}][\x{1F3FB}-\x{1F3FF}]?|\x{1F468}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D})?\x{1F468}|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}]|\x{1F466}(?:\x{200D}\x{1F466})?|\x{1F467}(?:\x{200D}[\x{1F466}\x{1F467}])?|[\x{1F468}\x{1F469}]\x{200D}(?:\x{1F466}(?:\x{200D}\x{1F466})?|\x{1F467}(?:\x{200D}[\x{1F466}\x{1F467}])?)|[\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}])|\x{1F3FB}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D})?\x{1F468}[\x{1F3FB}-\x{1F3FF}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F468}[\x{1F3FC}-\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FC}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D})?\x{1F468}[\x{1F3FB}-\x{1F3FF}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F468}[\x{1F3FB}\x{1F3FD}-\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FD}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D})?\x{1F468}[\x{1F3FB}-\x{1F3FF}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F468}[\x{1F3FB}\x{1F3FC}\x{1F3FE}\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FE}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D})?\x{1F468}[\x{1F3FB}-\x{1F3FF}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F468}[\x{1F3FB}-\x{1F3FD}\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FF}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D})?\x{1F468}[\x{1F3FB}-\x{1F3FF}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F468}[\x{1F3FB}-\x{1F3FE}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?)?|\x{1F469}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D})?[\x{1F468}\x{1F469}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}]|\x{1F466}(?:\x{200D}\x{1F466})?|\x{1F467}(?:\x{200D}[\x{1F466}\x{1F467}])?|\x{1F469}\x{200D}(?:\x{1F466}(?:\x{200D}\x{1F466})?|\x{1F467}(?:\x{200D}[\x{1F466}\x{1F467}])?)|[\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}])|\x{1F3FB}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FF}]|\x{1F48B}\x{200D}[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FF}])|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}[\x{1F468}\x{1F469}][\x{1F3FC}-\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FC}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FF}]|\x{1F48B}\x{200D}[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FF}])|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}[\x{1F468}\x{1F469}][\x{1F3FB}\x{1F3FD}-\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FD}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FF}]|\x{1F48B}\x{200D}[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FF}])|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}[\x{1F468}\x{1F469}][\x{1F3FB}\x{1F3FC}\x{1F3FE}\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FE}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FF}]|\x{1F48B}\x{200D}[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FF}])|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FD}\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FF}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FF}]|\x{1F48B}\x{200D}[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FF}])|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FE}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?)?|\x{1F46A}|[\x{1F46B}-\x{1F46D}][\x{1F3FB}-\x{1F3FF}]?|\x{1F46E}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F46F}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?|[\x{1F470}\x{1F471}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F472}[\x{1F3FB}-\x{1F3FF}]?|\x{1F473}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F474}-\x{1F476}][\x{1F3FB}-\x{1F3FF}]?|\x{1F477}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F478}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F479}-\x{1F47B}]|\x{1F47C}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F47D}-\x{1F480}]|[\x{1F481}\x{1F482}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F483}[\x{1F3FB}-\x{1F3FF}]?|\x{1F484}|\x{1F485}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F486}\x{1F487}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F488}-\x{1F48E}]|\x{1F48F}[\x{1F3FB}-\x{1F3FF}]?|\x{1F490}|\x{1F491}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F492}-\x{1F4A9}]|\x{1F4AA}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F4AB}-\x{1F4FC}]|\x{1F4FD}\x{FE0F}?|[\x{1F4FF}-\x{1F53D}]|[\x{1F549}\x{1F54A}]\x{FE0F}?|[\x{1F54B}-\x{1F54E}\x{1F550}-\x{1F567}]|[\x{1F56F}\x{1F570}\x{1F573}]\x{FE0F}?|\x{1F574}[\x{FE0F}\x{1F3FB}-\x{1F3FF}]?|\x{1F575}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{FE0F}\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F576}-\x{1F579}]\x{FE0F}?|\x{1F57A}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F587}\x{1F58A}-\x{1F58D}]\x{FE0F}?|\x{1F590}[\x{FE0F}\x{1F3FB}-\x{1F3FF}]?|[\x{1F595}\x{1F596}][\x{1F3FB}-\x{1F3FF}]?|\x{1F5A4}|[\x{1F5A5}\x{1F5A8}\x{1F5B1}\x{1F5B2}\x{1F5BC}\x{1F5C2}-\x{1F5C4}\x{1F5D1}-\x{1F5D3}\x{1F5DC}-\x{1F5DE}\x{1F5E1}\x{1F5E3}\x{1F5E8}\x{1F5EF}\x{1F5F3}\x{1F5FA}]\x{FE0F}?|[\x{1F5FB}-\x{1F62D}]|\x{1F62E}(?:\x{200D}\x{1F4A8})?|[\x{1F62F}-\x{1F634}]|\x{1F635}(?:\x{200D}\x{1F4AB})?|\x{1F636}(?:\x{200D}\x{1F32B}\x{FE0F}?)?|[\x{1F637}-\x{1F644}]|[\x{1F645}-\x{1F647}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F648}-\x{1F64A}]|\x{1F64B}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F64C}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F64D}\x{1F64E}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F64F}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F680}-\x{1F6A2}]|\x{1F6A3}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F6A4}-\x{1F6B3}]|[\x{1F6B4}-\x{1F6B6}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F6B7}-\x{1F6BF}]|\x{1F6C0}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F6C1}-\x{1F6C5}]|\x{1F6CB}\x{FE0F}?|\x{1F6CC}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F6CD}-\x{1F6CF}]\x{FE0F}?|[\x{1F6D0}-\x{1F6D2}\x{1F6D5}-\x{1F6D7}\x{1F6DD}-\x{1F6DF}]|[\x{1F6E0}-\x{1F6E5}\x{1F6E9}]\x{FE0F}?|[\x{1F6EB}\x{1F6EC}]|[\x{1F6F0}\x{1F6F3}]\x{FE0F}?|[\x{1F6F4}-\x{1F6FC}\x{1F7E0}-\x{1F7EB}\x{1F7F0}]|\x{1F90C}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F90D}\x{1F90E}]|\x{1F90F}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F910}-\x{1F917}]|[\x{1F918}-\x{1F91F}][\x{1F3FB}-\x{1F3FF}]?|[\x{1F920}-\x{1F925}]|\x{1F926}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F927}-\x{1F92F}]|[\x{1F930}-\x{1F934}][\x{1F3FB}-\x{1F3FF}]?|\x{1F935}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F936}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F937}-\x{1F939}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F93A}|\x{1F93C}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?|[\x{1F93D}\x{1F93E}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F93F}-\x{1F945}\x{1F947}-\x{1F976}]|\x{1F977}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F978}-\x{1F9B4}]|[\x{1F9B5}\x{1F9B6}][\x{1F3FB}-\x{1F3FF}]?|\x{1F9B7}|[\x{1F9B8}\x{1F9B9}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F9BA}|\x{1F9BB}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F9BC}-\x{1F9CC}]|[\x{1F9CD}-\x{1F9CF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F9D0}|\x{1F9D1}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F384}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F9D1}|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}])|\x{1F3FB}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D}|)\x{1F9D1}[\x{1F3FC}-\x{1F3FF}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F384}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F9D1}[\x{1F3FB}-\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FC}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D}|)\x{1F9D1}[\x{1F3FB}\x{1F3FD}-\x{1F3FF}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F384}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F9D1}[\x{1F3FB}-\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FD}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D}|)\x{1F9D1}[\x{1F3FB}\x{1F3FC}\x{1F3FE}\x{1F3FF}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F384}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F9D1}[\x{1F3FB}-\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FE}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D}|)\x{1F9D1}[\x{1F3FB}-\x{1F3FD}\x{1F3FF}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F384}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F9D1}[\x{1F3FB}-\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FF}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D}|)\x{1F9D1}[\x{1F3FB}-\x{1F3FE}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F384}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F9D1}[\x{1F3FB}-\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?)?|[\x{1F9D2}\x{1F9D3}][\x{1F3FB}-\x{1F3FF}]?|\x{1F9D4}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F9D5}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F9D6}-\x{1F9DD}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F9DE}\x{1F9DF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?|[\x{1F9E0}-\x{1F9FF}\x{1FA70}-\x{1FA74}\x{1FA78}-\x{1FA7C}\x{1FA80}-\x{1FA86}\x{1FA90}-\x{1FAAC}\x{1FAB0}-\x{1FABA}\x{1FAC0}-\x{1FAC2}]|[\x{1FAC3}-\x{1FAC5}][\x{1F3FB}-\x{1F3FF}]?|[\x{1FAD0}-\x{1FAD9}\x{1FAE0}-\x{1FAE7}]|\x{1FAF0}[\x{1F3FB}-\x{1F3FF}]?|\x{1FAF1}(?:\x{1F3FB}(?:\x{200D}\x{1FAF2}[\x{1F3FC}-\x{1F3FF}])?|\x{1F3FC}(?:\x{200D}\x{1FAF2}[\x{1F3FB}\x{1F3FD}-\x{1F3FF}])?|\x{1F3FD}(?:\x{200D}\x{1FAF2}[\x{1F3FB}\x{1F3FC}\x{1F3FE}\x{1F3FF}])?|\x{1F3FE}(?:\x{200D}\x{1FAF2}[\x{1F3FB}-\x{1F3FD}\x{1F3FF}])?|\x{1F3FF}(?:\x{200D}\x{1FAF2}[\x{1F3FB}-\x{1F3FE}])?)?|[\x{1FAF2}-\x{1FAF6}][\x{1F3FB}-\x{1F3FF}]?`) diff --git a/app/bot/spam_test.go b/app/bot/spam_test.go index f1ab4317..8c94c841 100644 --- a/app/bot/spam_test.go +++ b/app/bot/spam_test.go @@ -1,400 +1,336 @@ package bot import ( - "bytes" "context" + "errors" "io" - "net/http" - "strings" - "sync" + "os" + "path/filepath" + "strconv" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/umputun/tg-spam/app/bot/mocks" + "github.com/umputun/tg-spam/lib" ) -func TestFilter_OnMessage(t *testing.T) { - mockedHTTPClient := &mocks.HTTPClient{ - DoFunc: func(req *http.Request) (*http.Response, error) { - if strings.Contains(req.URL.String(), "101") { - return &http.Response{ - StatusCode: 200, - Body: io.NopCloser(bytes.NewBufferString(`{"ok": true, "description": "Is a spammer"}`)), - }, nil +func TestSpamFilter_OnMessage(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + det := &mocks.DetectorMock{ + CheckFunc: func(msg string, userID int64) (bool, []lib.CheckResult) { + if msg == "spam" { + return true, []lib.CheckResult{{Name: "something", Spam: true, Details: "some spam"}} } - return &http.Response{ - StatusCode: 200, - Body: io.NopCloser(bytes.NewBufferString(`{"ok": false, "description": "Not a spammer"}`)), - }, nil + return false, nil }, } - filter, err := NewSpamFilter(context.Background(), SpamParams{ - SpamSamplesFile: "testdata/spam-samples.txt", // "win free iPhone\nlottery prize - HamSamplesFile: "testdata/ham-samples.txt", - StopWordsFile: "testdata/stop-words.txt", - ExcludedTokensFile: "testdata/spam-exclude-token.txt", - SimilarityThreshold: 0.5, - MinMsgLen: 5, - Dry: false, - HTTPClient: mockedHTTPClient, - SpamMsg: "this is spam! go to ban", - MaxAllowedEmoji: 2, + t.Run("spam detected", func(t *testing.T) { + s := NewSpamFilter(ctx, det, SpamConfig{SpamMsg: "detected", SpamDryMsg: "detected dry"}) + resp := s.OnMessage(Message{Text: "spam", From: User{ID: 1, Username: "john"}}) + assert.Equal(t, Response{Text: `detected: "john" (1)`, Send: true, BanInterval: PermanentBanDuration, + User: User{ID: 1, Username: "john"}, DeleteReplyTo: true}, resp) + t.Logf("resp: %+v", resp) }) - require.NoError(t, err) + + t.Run("spam detected, dry", func(t *testing.T) { + s := NewSpamFilter(ctx, det, SpamConfig{SpamMsg: "detected", SpamDryMsg: "detected dry", Dry: true}) + resp := s.OnMessage(Message{Text: "spam", From: User{ID: 1, Username: "john"}}) + assert.Equal(t, `detected dry: "john" (1)`, resp.Text) + assert.True(t, resp.Send) + }) + + t.Run("ham detected", func(t *testing.T) { + s := NewSpamFilter(ctx, det, SpamConfig{SpamMsg: "detected", SpamDryMsg: "detected dry"}) + resp := s.OnMessage(Message{Text: "good", From: User{ID: 1, Username: "john"}}) + assert.Equal(t, Response{}, resp) + }) + +} + +func TestSpamFilter_reloadSamples(t *testing.T) { + mockDirector := &mocks.DetectorMock{ + LoadSamplesFunc: func(exclReader io.Reader, spamReaders []io.Reader, hamReaders []io.Reader) (lib.LoadResult, error) { + return lib.LoadResult{}, nil + }, + LoadStopWordsFunc: func(readers ...io.Reader) (lib.LoadResult, error) { + return lib.LoadResult{}, nil + }, + } tests := []struct { - name string - msg Message - expected Response + name string + modify func(s *SpamConfig) + expectedErr error }{ { - "good message", - Message{From: User{ID: 1, Username: "john", DisplayName: "John"}, Text: "Hello, how are you?", ID: 1}, - Response{}, + name: "Successful execution", + modify: func(s *SpamConfig) {}, + expectedErr: nil, }, { - "emoji spam", - Message{From: User{ID: 4, Username: "john", DisplayName: "John"}, Text: "Hello 😁🐶🍕 how are you? ", ID: 4}, - Response{Text: "this is spam! go to ban: \"John\" (4)", Send: true, - BanInterval: permanentBanDuration, ReplyTo: 4, DeleteReplyTo: true, - User: User{ID: 4, Username: "john", DisplayName: "John"}}, + name: "Spam samples file open failure", + modify: func(s *SpamConfig) { + s.SpamSamplesFile = "fail" + }, + expectedErr: errors.New("failed to open spam samples file \"fail\": open fail: no such file or directory"), }, { - "similarity spam", - Message{From: User{ID: 2, Username: "spammer", DisplayName: "Spammer"}, Text: "Win a free iPhone now!", ID: 2}, - Response{Text: "this is spam! go to ban: \"Spammer\" (2)", Send: true, - ReplyTo: 2, BanInterval: permanentBanDuration, DeleteReplyTo: true, - User: User{ID: 2, Username: "spammer", DisplayName: "Spammer"}, + name: "Ham samples file open failure", + modify: func(s *SpamConfig) { + s.HamSamplesFile = "fail" }, + expectedErr: errors.New("failed to open ham samples file \"fail\": open fail: no such file or directory"), }, { - "classifier spam", - Message{From: User{ID: 2, Username: "spammer", DisplayName: "Spammer"}, Text: "free gift for you", ID: 2}, - Response{Text: "this is spam! go to ban: \"Spammer\" (2)", Send: true, - ReplyTo: 2, BanInterval: permanentBanDuration, DeleteReplyTo: true, - User: User{ID: 2, Username: "spammer", DisplayName: "Spammer"}, + name: "Stop words file not found", + modify: func(s *SpamConfig) { + s.StopWordsFile = "notfound" }, + expectedErr: nil, }, { - "CAS spam", - Message{From: User{ID: 101, Username: "spammer", DisplayName: "blah"}, Text: "something something", ID: 10}, - Response{Text: "this is spam! go to ban: \"blah\" (101)", Send: true, - ReplyTo: 10, BanInterval: permanentBanDuration, DeleteReplyTo: true, - User: User{ID: 101, Username: "spammer", DisplayName: "blah"}, + name: "Excluded tokens file not found", + modify: func(s *SpamConfig) { + s.ExcludedTokensFile = "notfound" }, + expectedErr: nil, }, { - "stop words spam emoji", - Message{From: User{ID: 102, Username: "spammer", DisplayName: "blah"}, Text: "something пишите в лс something", ID: 10}, - Response{Text: "this is spam! go to ban: \"blah\" (102)", Send: true, - ReplyTo: 10, BanInterval: permanentBanDuration, DeleteReplyTo: true, - User: User{ID: 102, Username: "spammer", DisplayName: "blah"}, + name: "Spam dynamic file not found", + modify: func(s *SpamConfig) { + s.SpamDynamicFile = "notfound" }, + expectedErr: nil, + }, + { + name: "Ham dynamic file not found", + modify: func(s *SpamConfig) { + s.HamDynamicFile = "notfound" + }, + expectedErr: nil, }, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.expected, filter.OnMessage(test.msg)) - }) - } -} - -func TestIsSpam(t *testing.T) { - spamSamples := strings.NewReader("win free iPhone\nlottery prize") - filter := SpamFilter{ - tokenizedSpam: []map[string]int{}, - lock: sync.RWMutex{}, - } - - tests := []struct { - name string - message string - threshold float64 - expected bool - }{ - {"Not Spam", "Hello, how are you?", 0.5, false}, - {"Exact Match", "Win a free iPhone now!", 0.5, true}, - {"Similar Match", "You won a lottery prize!", 0.3, true}, - {"High Threshold", "You won a lottery prize!", 0.9, false}, - {"Partial Match", "win free", 0.9, false}, - {"Low Threshold", "win free", 0.8, true}, - } - - err := filter.loadSpamSamples(spamSamples) - require.NoError(t, err) - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - filter.SimilarityThreshold = test.threshold // Update threshold for each test case - assert.Equal(t, test.expected, filter.isSpamSimilarityHigh(test.message)) - }) - } -} - -// nolint -func TestTooManyEmojis(t *testing.T) { - filter := SpamFilter{ - SpamParams: SpamParams{MaxAllowedEmoji: 2}, - } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Create temporary files for each test + spamSamplesFile, err := os.CreateTemp("", "spam") + require.NoError(t, err) + defer os.Remove(spamSamplesFile.Name()) + + hamSamplesFile, err := os.CreateTemp("", "ham") + require.NoError(t, err) + defer os.Remove(hamSamplesFile.Name()) + + stopWordsFile, err := os.CreateTemp("", "stopwords") + require.NoError(t, err) + defer os.Remove(stopWordsFile.Name()) + + excludedTokensFile, err := os.CreateTemp("", "excludedtokens") + require.NoError(t, err) + defer os.Remove(excludedTokensFile.Name()) + + // Reset to default values before each test + params := SpamConfig{ + SpamSamplesFile: spamSamplesFile.Name(), + HamSamplesFile: hamSamplesFile.Name(), + StopWordsFile: stopWordsFile.Name(), + ExcludedTokensFile: excludedTokensFile.Name(), + SpamDynamicFile: "optional", + HamDynamicFile: "optional", + } + tc.modify(¶ms) + s := NewSpamFilter(ctx, mockDirector, params) - tests := []struct { - name string - input string - count int - spam bool - }{ - {"NoEmoji", "Hello, world!", 0, false}, - {"OneEmoji", "Hi there 👋", 1, false}, - {"TwoEmojis", "Good morning 🌞🌻", 2, false}, - {"Mixed", "👨‍👩‍👧‍👦 Family emoji", 1, false}, - {"EmojiSequences", "🏳️‍🌈 Rainbow flag", 1, false}, - {"TextAfterEmoji", "😊 Have a nice day!", 1, false}, - {"OnlyEmojis", "😁🐶🍕", 3, true}, - {"WithCyrillic", "Привет 🌞 🍕 мир! 👋", 3, true}, - } + err = s.ReloadSamples() - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - isSpam, count := filter.tooManyEmojis(tt.input, 2) - assert.Equal(t, tt.count, count) - assert.Equal(t, tt.spam, isSpam) + if tc.expectedErr != nil { + require.Error(t, err) + assert.Equal(t, tc.expectedErr.Error(), err.Error()) + } else { + assert.NoError(t, err) + } }) } } -func TestStopWords(t *testing.T) { - filter := &SpamFilter{ - stopWords: []string{"в личку", "всем привет"}, - } +func TestSpamFilter_watch(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - tests := []struct { - name string - message string - expected bool - }{ - { - name: "Stop word present", - message: "Hello, please send me a message в личку", - expected: true, - }, - { - name: "Stop word present with emoji", - message: "👋Всем привет\nИщу амбициозного человека к се6е в команду\nКто в поисках дополнительного заработка или хочет попробовать себя в новой сфере деятельности! 👨🏻\u200d💻\nПишите в лс✍️", - expected: true, - }, - { - name: "No stop word present", - message: "Hello, how are you?", - expected: false, + count := 0 + mockDetector := &mocks.DetectorMock{ + LoadSamplesFunc: func(exclReader io.Reader, spamReaders []io.Reader, hamReaders []io.Reader) (lib.LoadResult, error) { + count++ + if count == 1 { // only first call should succeed + return lib.LoadResult{}, nil + } + return lib.LoadResult{}, errors.New("error") }, - { - name: "Case insensitive stop word present", - message: "Hello, please send me a message В ЛИЧКУ", - expected: true, + LoadStopWordsFunc: func(readers ...io.Reader) (lib.LoadResult, error) { + return lib.LoadResult{}, nil }, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.expected, filter.hasStopWords(test.message)) - }) - } -} + tmpDir, err := os.MkdirTemp("", "spamfilter_test") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) -func TestSpam_isCasSpam(t *testing.T) { + excludedTokensFile := filepath.Join(tmpDir, "excluded_tokens.txt") + spamSamplesFile := filepath.Join(tmpDir, "spam_samples.txt") + hamSamplesFile := filepath.Join(tmpDir, "ham_samples.txt") + stopWordsFile := filepath.Join(tmpDir, "stop_words.txt") - tests := []struct { - name string - mockResp string - mockStatusCode int - expected bool - }{ - { - name: "User is not a spammer", - mockResp: `{"ok": false, "description": "Not a spammer"}`, - mockStatusCode: 200, - expected: false, - }, - { - name: "User is a spammer", - mockResp: `{"ok": true, "description": "Is a spammer"}`, - mockStatusCode: 200, - expected: true, - }, - { - name: "HTTP error", - mockResp: "", - mockStatusCode: 500, - expected: false, - }, - } + _, err = os.Create(excludedTokensFile) + require.NoError(t, err) + _, err = os.Create(spamSamplesFile) + require.NoError(t, err) + _, err = os.Create(hamSamplesFile) + require.NoError(t, err) + _, err = os.Create(stopWordsFile) + require.NoError(t, err) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - mockedHTTPClient := &mocks.HTTPClient{ - DoFunc: func(req *http.Request) (*http.Response, error) { - return &http.Response{ - StatusCode: tt.mockStatusCode, - Body: io.NopCloser(bytes.NewBufferString(tt.mockResp)), - }, nil - }, - } + NewSpamFilter(ctx, mockDetector, SpamConfig{ + ExcludedTokensFile: excludedTokensFile, + SpamSamplesFile: spamSamplesFile, + HamSamplesFile: hamSamplesFile, + StopWordsFile: stopWordsFile, + WatchDelay: time.Millisecond * 100, + }) - s := SpamFilter{SpamParams: SpamParams{ - CasAPI: "http://localhost", - HTTPClient: mockedHTTPClient, - }} - - msg := Message{ - From: User{ - ID: 1, - Username: "testuser", - DisplayName: "Test User", - }, - ID: 1, - Text: "Hello", - } + time.Sleep(200 * time.Millisecond) // let it start - isSpam := s.isCasSpam(msg.From.ID) - assert.Equal(t, tt.expected, isSpam) - }) - } -} + assert.Equal(t, 0, len(mockDetector.LoadSamplesCalls())) + assert.Equal(t, 0, len(mockDetector.LoadStopWordsCalls())) -func Test_tokenChan(t *testing.T) { - tests := []struct { - name string - input string - expected []string - }{ - {name: "empty", input: "", expected: []string{}}, - {name: "token per line", input: "hello\nworld", expected: []string{"hello", "world"}}, - {name: "token per line", input: "hello 123\nworld", expected: []string{"hello 123", "world"}}, - {name: "token per line with spaces", input: "hello \n world", expected: []string{"hello", "world"}}, - {name: "tokens comma separated", input: "\"hello\",\"world\"\nsomething", expected: []string{"hello", "world", "something"}}, - {name: "tokens comma separated, extra EOL", input: "\"hello\",world\nsomething\n", expected: []string{"hello", "world", "something"}}, - {name: "tokens comma separated, empty tokens", input: "\"hello\",world,\"\"\nsomething\n ", expected: []string{"hello", "world", "something"}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ch := tokenChan(bytes.NewBufferString(tt.input)) - res := []string{} - for token := range ch { - res = append(res, token) - } - assert.Equal(t, tt.expected, res) - }) - } -} + // write to spam samples file + message := "spam message" + err = os.WriteFile(spamSamplesFile, []byte(message), 0o600) + require.NoError(t, err) + // wait for reload to complete + time.Sleep(time.Millisecond * 200) -func TestSpamFilter_tokenize(t *testing.T) { - tests := []struct { - name string - input string - expected map[string]int - }{ - {name: "empty", input: "", expected: map[string]int{}}, - {name: "no filters or cleanups", input: "hello world", expected: map[string]int{"hello": 1, "world": 1}}, - {name: "with excluded tokens", input: "hello world the she", expected: map[string]int{"hello": 1, "world": 1}}, - {name: "with short tokens", input: "hello world the she a or", expected: map[string]int{"hello": 1, "world": 1}}, - {name: "with repeated tokens", input: "hello world hello world", expected: map[string]int{"hello": 2, "world": 2}}, - } + assert.Equal(t, 1, len(mockDetector.LoadSamplesCalls())) + assert.Equal(t, 1, len(mockDetector.LoadStopWordsCalls())) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := SpamFilter{ - excludedTokens: []string{"the", "she"}, - } - assert.Equal(t, tt.expected, s.tokenize(tt.input)) - }) - } + // write to ham samples file + message = "ham message" + err = os.WriteFile(hamSamplesFile, []byte(message), 0o600) + require.NoError(t, err) + // wait for reload to complete + time.Sleep(time.Millisecond * 200) + assert.Equal(t, 2, len(mockDetector.LoadSamplesCalls())) + assert.Equal(t, 1, len(mockDetector.LoadStopWordsCalls())) + + // wait to make sure no more reloads happen + time.Sleep(time.Millisecond * 500) + assert.Equal(t, 2, len(mockDetector.LoadSamplesCalls())) + assert.Equal(t, 1, len(mockDetector.LoadStopWordsCalls())) } -func TestSpamFilter_UpdateSpam(t *testing.T) { +func TestSpamFilter_WatchMultipleUpdates(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - upd := &mocks.SampleUpdater{ - AppendFunc: func(msg string) error { - return nil + mockDetector := &mocks.DetectorMock{ + LoadSamplesFunc: func(exclReader io.Reader, spamReaders []io.Reader, hamReaders []io.Reader) (lib.LoadResult, error) { + return lib.LoadResult{}, nil + }, + LoadStopWordsFunc: func(readers ...io.Reader) (lib.LoadResult, error) { + return lib.LoadResult{}, nil }, } - s := SpamFilter{ - spamSamplesUpd: upd, - tokenizedSpam: []map[string]int{}, - spamClassifier: Classifier{}, - } - s.spamClassifier.Reset() - - err := s.UpdateSpam("some spam message") + tmpDir, err := os.MkdirTemp("", "spamfilter_test") require.NoError(t, err) - assert.Equal(t, []map[string]int{{"some": 1, "spam": 1, "message": 1}}, s.tokenizedSpam) - assert.Equal(t, 1, s.spamClassifier.NAllDocument) + defer os.RemoveAll(tmpDir) + + excludedTokensFile := filepath.Join(tmpDir, "excluded_tokens.txt") + spamSamplesFile := filepath.Join(tmpDir, "spam_samples.txt") + hamSamplesFile := filepath.Join(tmpDir, "ham_samples.txt") + stopWordsFile := filepath.Join(tmpDir, "stop_words.txt") - err = s.UpdateSpam("more things") + _, err = os.Create(excludedTokensFile) + require.NoError(t, err) + _, err = os.Create(spamSamplesFile) + require.NoError(t, err) + _, err = os.Create(hamSamplesFile) + require.NoError(t, err) + _, err = os.Create(stopWordsFile) require.NoError(t, err) - assert.Equal(t, []map[string]int{{"some": 1, "spam": 1, "message": 1}, {"more": 1, "things": 1}}, s.tokenizedSpam) - assert.Equal(t, 2, s.spamClassifier.NAllDocument) -} -func TestSpamFilter_UpdateHam(t *testing.T) { + NewSpamFilter(ctx, mockDetector, SpamConfig{ + ExcludedTokensFile: excludedTokensFile, + SpamSamplesFile: spamSamplesFile, + HamSamplesFile: hamSamplesFile, + StopWordsFile: stopWordsFile, + WatchDelay: time.Millisecond * 100, + }) - upd := &mocks.SampleUpdater{ - AppendFunc: func(msg string) error { - return nil - }, - } + time.Sleep(200 * time.Millisecond) // let it start - s := SpamFilter{ - hamSamplesUpd: upd, - tokenizedSpam: []map[string]int{}, - spamClassifier: Classifier{}, + // simulate rapid file changes + message := "spam message" + for i := 0; i < 5; i++ { + err = os.WriteFile(spamSamplesFile, []byte(message+strconv.Itoa(i)), 0o600) + require.NoError(t, err) + time.Sleep(10 * time.Millisecond) // less than the debounce interval } - s.spamClassifier.Reset() - err := s.UpdateHam("some spam message") - require.NoError(t, err) - assert.Equal(t, []map[string]int{}, s.tokenizedSpam) - assert.Equal(t, 1, s.spamClassifier.NAllDocument) - assert.Equal(t, 1, len(upd.AppendCalls())) + // wait for reload to complete + time.Sleep(200 * time.Millisecond) - err = s.UpdateHam("more things") - require.NoError(t, err) - assert.Equal(t, []map[string]int{}, s.tokenizedSpam) - assert.Equal(t, 2, s.spamClassifier.NAllDocument) - assert.Equal(t, 2, len(upd.AppendCalls())) + // ponly one reload should happen despite multiple updates + assert.Equal(t, 1, len(mockDetector.LoadSamplesCalls())) + + // make sure no more reloads happen + time.Sleep(500 * time.Millisecond) + assert.Equal(t, 1, len(mockDetector.LoadSamplesCalls())) } -func TestSpamFilter_loadDynFiles(t *testing.T) { - count := 0 - upd := &mocks.SampleUpdater{ - AppendFunc: func(msg string) error { +func TestSpamFilter_Update(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + mockDetector := &mocks.DetectorMock{ + UpdateSpamFunc: func(msg string) error { + if msg == "err" { + return errors.New("error") + } return nil }, - ReaderFunc: func() (io.ReadCloser, error) { - count++ - if count <= 2 { // two calls, for spam, one for ham - return io.NopCloser(bytes.NewBufferString("spam1 spam2 spam3\nspam4")), nil + UpdateHamFunc: func(msg string) error { + if msg == "err" { + return errors.New("error") } - return io.NopCloser(bytes.NewBufferString("ham1\nham2\nham3")), nil + return nil }, } - s := SpamFilter{ - spamSamplesUpd: upd, - hamSamplesUpd: upd, - tokenizedSpam: []map[string]int{}, - spamClassifier: Classifier{}, - } - s.spamClassifier.Reset() + sf := NewSpamFilter(ctx, mockDetector, SpamConfig{}) - err := s.loadDynFiles() - require.NoError(t, err) - assert.Equal(t, []map[string]int{{"spam1": 1, "spam2": 1, "spam3": 1}, {"spam4": 1}}, s.tokenizedSpam) - assert.Equal(t, 2+3, s.spamClassifier.NAllDocument) - assert.Equal(t, 0, len(upd.AppendCalls())) - assert.Equal(t, 2+1, len(upd.ReaderCalls())) + t.Run("good update", func(t *testing.T) { + err := sf.UpdateSpam("spam") + assert.NoError(t, err) + + err = sf.UpdateHam("ham") + assert.NoError(t, err) + }) + + t.Run("bad update", func(t *testing.T) { + err := sf.UpdateSpam("err") + assert.Error(t, err) + + err = sf.UpdateHam("err") + assert.Error(t, err) + }) } diff --git a/app/bot/testdata/ham-samples.txt b/app/bot/testdata/ham-samples.txt deleted file mode 100644 index 7d799c57..00000000 --- a/app/bot/testdata/ham-samples.txt +++ /dev/null @@ -1,2 +0,0 @@ -this is a good one -hello world diff --git a/app/bot/testdata/spam-exclude-token.txt b/app/bot/testdata/spam-exclude-token.txt deleted file mode 100644 index 5f0fc454..00000000 --- a/app/bot/testdata/spam-exclude-token.txt +++ /dev/null @@ -1,33 +0,0 @@ -"and", "the", "is", "in", "on", "at", "for", "with", "not", -"by", "be", "this", "are", "from", "or", "that", "an", "it", -"his", "but", "he", "she", "as", "you", "do", "their", "all", -"will", "there", "can", "i", "me", "my", "myself", "we", "our", "ours", "ourselves", -"your", "yours", "yourself", "yourselves", "him", "himself", -"her", "hers", "herself", "its", "itself", "they", "them", -"theirs", "themselves", "what", "which", "who", "whom", -"these", "those", "am", "was", "were", "been", "being", "have", -"has", "had", "having", "does", "did", "a", "if", "because", -"until", "while", "of", "about", "against", "between", "into", -"through", "during", "before", "after", "above", "below", "to", -"up", "down", "out", "off", "over", "under", "again", "further", -"then", "once", "here", "why", "how", "any", "both", "each", -"few", "more", "most", "other", "some", "such", "no", "nor", -"only", "own", "same", "so", "than", "too", "very", "s", "t", -"just", "don", "should", "now", - -"а", "без", "более", "больше", "будет", "будто", "бы", "был", "была", "были", -"было", "быть", "в", "вам", "вас", "вдруг", "ведь", "во", "вот", "впрочем", -"все", "всегда", "всего", "всех", "всю", "вы", "где", "да", "даже", "два", -"для", "до", "другой", "его", "ее", "если", "есть", "еще", "же", "за", "здесь", -"и", "из", "или", "им", "иногда", "их", "к", "как", "какая", "какой", "когда", -"конечно", "которого", "которые", "кто", "куда", "ли", "лучше", "между", -"меня", "мне", "много", "может", "можно", "мой", "моя", "мы", "на", "над", -"надо", "наконец", "нас", "не", "него", "нее", "нельзя", "нет", "ни", "нибудь", -"никогда", "ним", "них", "ничего", "но", "ну", "о", "об", "один", "он", "она", -"они", "оно", "опять", "от", "перед", "по", "под", "после", "потом", "потому", -"почти", "при", "про", "раз", "разве", "с", "сам", "свое", "свою", "себе", -"себя", "сегодня", "сейчас", "сказал", "сказала", "сказать", "со", "совсем", -"так", "такой", "там", "тебя", "тем", "теперь", "то", "тогда", "того", "тоже", -"только", "том", "тот", "три", "тут", "ты", "у", "уж", "уже", "хорошо", "хоть", -"чего", "чей", "чем", "через", "что", "чтоб", "чтобы", "чуть", "эти", "этого", -"этой", "этом", "этот", "эту", "я", \ No newline at end of file diff --git a/app/bot/testdata/spam-samples.txt b/app/bot/testdata/spam-samples.txt deleted file mode 100644 index b5543575..00000000 --- a/app/bot/testdata/spam-samples.txt +++ /dev/null @@ -1,2 +0,0 @@ -win free iPhone -lottery prize diff --git a/app/bot/testdata/stop-words.txt b/app/bot/testdata/stop-words.txt deleted file mode 100644 index 5e2698ee..00000000 --- a/app/bot/testdata/stop-words.txt +++ /dev/null @@ -1,9 +0,0 @@ -в личку -писать в лc -пишите в лс -в лuчные сообщенuя -личных сообщениях -заработок удалённо -заработок в интернете -заработок в сети -для yдaлённoгo зaрaбoткa diff --git a/app/events/telegram.go b/app/events/events.go similarity index 75% rename from app/events/telegram.go rename to app/events/events.go index 4a065253..8a44a51f 100644 --- a/app/events/telegram.go +++ b/app/events/events.go @@ -1,3 +1,9 @@ +// Package events provide event handlers for telegram bot and all the high-level event handlers. +// It parses messages, sends them to the spam detector and handles the results. It can also ban users +// and send messages to the admin. +// +// In addition to that, it provides support for admin chat handling allowing to unban users via the web service and +// update the list of spam samples. package events import ( @@ -9,8 +15,8 @@ import ( "strings" "sync" "time" + "unicode/utf8" - "github.com/go-pkgz/notify" tbapi "github.com/go-telegram-bot-api/telegram-bot-api/v5" "github.com/hashicorp/go-multierror" @@ -37,6 +43,7 @@ type TelegramListener struct { NoSpamReply bool Dry bool SpamWeb SpamWeb + Locator *Locator chatID int64 adminChatID int64 @@ -157,28 +164,21 @@ func (l *TelegramListener) procEvents(update tbapi.Update) error { msg := l.transform(update.Message) fromChat := update.Message.Chat.ID + // message from admin chat if l.isAdminChat(fromChat, msg.From.Username) { - // message from supers to admin chat - if update.Message.ForwardSenderName != "" { - // this is a forwarded message from super to admin chat, it is an example of missed spam - // we need to update spam filter with this message - if err := l.Bot.UpdateSpam(strings.ReplaceAll(update.Message.Text, "\n", " ")); err != nil { - log.Printf("[WARN] failed to update spam for %q, %v", update.Message.Text, err) - return nil - } - log.Printf("[DEBUG] spam updated with %q", update.Message.Text) - // it would be nice to ban this user right away, but we don't have forwarded user ID here, it is empty in update.Message + if err := l.adminChatMsgHandler(update, fromChat); err != nil { + log.Printf("[WARN] failed to process admin chat message: %v", err) } return nil } + // ignore messages from other chats if not in the test list if !l.isChatAllowed(fromChat) { - // ignore messages from other chats if not in the test list return nil } log.Printf("[DEBUG] incoming msg: %+v", strings.ReplaceAll(msg.Text, "\n", " ")) - + l.Locator.Add(update.Message.Text, fromChat, msg.From.ID, msg.ID) // save message to locator resp := l.Bot.OnMessage(*msg) if resp.Send && !l.NoSpamReply { @@ -188,9 +188,9 @@ func (l *TelegramListener) procEvents(update tbapi.Update) error { } errs := new(multierror.Error) - isBanInvoked := resp.Send && resp.BanInterval > 0 - // some bots may request a direct ban for given duration - if isBanInvoked { + + // ban user if requested by bot + if resp.Send && resp.BanInterval > 0 { log.Printf("[DEBUG] ban initiated for %+v", resp) l.SpamLogger.Save(msg, &resp) banUserStr := l.getBanUsername(resp, update) @@ -198,27 +198,76 @@ func (l *TelegramListener) procEvents(update tbapi.Update) error { log.Printf("[DEBUG] superuser %s requested ban, ignored", banUserStr) return nil } - banSuccessMessage := fmt.Sprintf("[INFO] %s banned by bot for %v", banUserStr, resp.BanInterval) - if err := l.banUserOrChannel(resp.BanInterval, fromChat, resp.User.ID, resp.ChannelID); err != nil { - errs = multierror.Append(errs, fmt.Errorf("failed to ban %s: %w", banUserStr, err)) - } else { - log.Print(banSuccessMessage) + if err := l.banUserOrChannel(resp.BanInterval, fromChat, resp.User.ID, resp.ChannelID); err == nil { + log.Printf("[INFO] %s banned by bot for %v", banUserStr, resp.BanInterval) if l.adminChatID != 0 && msg.From.ID != 0 { - l.forwardToAdmin(banUserStr, msg) + l.reportToAdminChat(banUserStr, msg) } + } else { + errs = multierror.Append(errs, fmt.Errorf("failed to ban %s: %w", banUserStr, err)) } } // delete message if requested by bot - if resp.DeleteReplyTo && resp.ReplyTo != 0 && !l.Dry { - _, err := l.TbAPI.Request(tbapi.DeleteMessageConfig{ChatID: l.chatID, MessageID: resp.ReplyTo}) - if err != nil { + if resp.DeleteReplyTo && resp.ReplyTo != 0 && !l.Dry && !l.SuperUsers.IsSuper(msg.From.Username) { + if _, err := l.TbAPI.Request(tbapi.DeleteMessageConfig{ChatID: l.chatID, MessageID: resp.ReplyTo}); err != nil { errs = multierror.Append(errs, fmt.Errorf("failed to delete message %d: %w", resp.ReplyTo, err)) } } + return errs.ErrorOrNil() } +// adminChatMsgHandler handles messages received on admin chat. This is usually forwarded spam failed +// to be detected by the bot. We need to update spam filter with this message and ban the user. +func (l *TelegramListener) adminChatMsgHandler(update tbapi.Update, fromChat int64) error { + shrink := func(inp string, max int) string { + if utf8.RuneCountInString(inp) <= max { + return inp + } + return string([]rune(inp)[:max]) + "..." + } + + // message from supers to admin chat + if update.Message.ForwardSenderName != "" || update.FromChat() != nil { + // this is a forwarded message from super to admin chat, it is an example of missed spam + // we need to update spam filter with this message + msgTxt := strings.ReplaceAll(update.Message.Text, "\n", " ") + log.Printf("[DEBUG] forwarded message from superuser %q to admin chat %d: %q", + update.Message.From.UserName, l.adminChatID, msgTxt) + + if !l.Dry { + if err := l.Bot.UpdateSpam(msgTxt); err != nil { + return fmt.Errorf("failed to update spam for %q: %w", msgTxt, err) + } + log.Printf("[INFO] spam updated with %q", shrink(update.Message.Text, 50)) + } + + // it would be nice to ban this user right away, but we don't have forwarded user ID here due to tg privacy limiatation, + // it is empty in update.Message. To ban this user, we need to get the match on the message from the locator and ban from there. + info, ok := l.Locator.Get(update.Message.Text) + if !ok { + return fmt.Errorf("not found %q in locator", shrink(update.Message.Text, 50)) + } + + log.Printf("[DEBUG] locator found message %s", info) + if l.Dry { + return nil + } + + if _, err := l.TbAPI.Request(tbapi.DeleteMessageConfig{ChatID: l.chatID, MessageID: info.msgID}); err != nil { + return fmt.Errorf("failed to delete message %d: %w", info.msgID, err) + } + log.Printf("[INFO] message %d deleted", info.msgID) + + if err := l.banUserOrChannel(bot.PermanentBanDuration, fromChat, info.userID, 0); err != nil { + return fmt.Errorf("failed to ban user %d: %w", info.userID, err) + } + log.Printf("[INFO] user %q (%d) banned", update.Message.ForwardSenderName, info.userID) + } + return nil +} + func (l *TelegramListener) isChatAllowed(fromChat int64) bool { if fromChat == l.chatID { return true @@ -235,7 +284,7 @@ func (l *TelegramListener) isAdminChat(fromChat int64, from string) bool { return fromChat == l.adminChatID && l.SuperUsers.IsSuper(from) } -func (l *TelegramListener) forwardToAdmin(banUserStr string, msg *bot.Message) { +func (l *TelegramListener) reportToAdminChat(banUserStr string, msg *bot.Message) { // escapeMarkDownV1Text escapes markdownV1 special characters, used in places where we want to send text as-is. // For example, telegram username with underscores would be italicized if we don't escape it. // https://core.telegram.org/bots/api#markdown-style @@ -247,13 +296,12 @@ func (l *TelegramListener) forwardToAdmin(banUserStr string, msg *bot.Message) { return text } - log.Printf("[DEBUG] forward to admin ban data for %s, group: %d", banUserStr, l.adminChatID) + log.Printf("[DEBUG] report to admin chat, ban data for %s, group: %d", banUserStr, l.adminChatID) text := strings.ReplaceAll(escapeMarkDownV1Text(msg.Text), "\n", " ") forwardMsg := fmt.Sprintf("**permanently banned [%s](tg://user?id=%d)**\n[⛔︎ unban if wrong ⛔︎](%s)\n\n%s\n\n", banUserStr, msg.From.ID, l.SpamWeb.UnbanURL(msg.From.ID), text) - e := l.sendBotResponse(bot.Response{Send: true, Text: forwardMsg, ParseMode: tbapi.ModeMarkdown}, l.adminChatID) - if e != nil { - log.Printf("[WARN] failed to send admin message, %v", e) + if err := l.sendBotResponse(bot.Response{Send: true, Text: forwardMsg, ParseMode: tbapi.ModeMarkdown}, l.adminChatID); err != nil { + log.Printf("[WARN] failed to send admin message, %v", err) } } @@ -296,32 +344,6 @@ func (l *TelegramListener) sendBotResponse(resp bot.Response, chatID int64) erro return nil } -// Submit message text to telegram's group -func (l *TelegramListener) Submit(ctx context.Context, text string) error { - l.msgs.once.Do(func() { l.msgs.ch = make(chan bot.Response, 100) }) - - select { - case <-ctx.Done(): - return ctx.Err() - case l.msgs.ch <- bot.Response{Text: text, Send: true}: - } - return nil -} - -// SubmitHTML message to telegram's group with HTML mode -func (l *TelegramListener) SubmitHTML(ctx context.Context, text string) error { - // Remove unsupported HTML tags - text = notify.TelegramSupportedHTML(text) - l.msgs.once.Do(func() { l.msgs.ch = make(chan bot.Response, 100) }) - - select { - case <-ctx.Done(): - return ctx.Err() - case l.msgs.ch <- bot.Response{Text: text, Send: true, ParseMode: tbapi.ModeHTML}: - } - return nil -} - func (l *TelegramListener) getChatID(group string) (int64, error) { chatID, err := strconv.ParseInt(group, 10, 64) if err == nil { @@ -490,3 +512,16 @@ func (l *TelegramListener) transformEntities(entities []tbapi.MessageEntity) *[] return &result } + +// SuperUser for moderators +type SuperUser []string + +// IsSuper checks if username in the list of super users +func (s SuperUser) IsSuper(userName string) bool { + for _, super := range s { + if strings.EqualFold(userName, super) || strings.EqualFold("/"+userName, super) { + return true + } + } + return false +} diff --git a/app/events/telegram_test.go b/app/events/events_test.go similarity index 90% rename from app/events/telegram_test.go rename to app/events/events_test.go index 66933109..20ce9f73 100644 --- a/app/events/telegram_test.go +++ b/app/events/events_test.go @@ -38,6 +38,7 @@ func TestTelegramListener_Do(t *testing.T) { Group: "gr", AdminGroup: "987654321", StartupMsg: "startup", + Locator: NewLocator(10 * time.Minute), } ctx, cancel := context.WithTimeout(context.Background(), 500*time.Minute) @@ -98,6 +99,7 @@ func TestTelegramListener_DoWithBotBan(t *testing.T) { Bot: b, SuperUsers: SuperUser{"admin"}, Group: "gr", + Locator: NewLocator(10 * time.Minute), } ctx, cancel := context.WithTimeout(context.Background(), 500*time.Minute) @@ -228,6 +230,7 @@ func TestTelegramListener_DoDeleteMessages(t *testing.T) { TbAPI: mockAPI, Bot: b, Group: "gr", + Locator: NewLocator(10 * time.Minute), } ctx, cancel := context.WithTimeout(context.Background(), 500*time.Minute) @@ -274,6 +277,9 @@ func TestTelegramListener_DoWithForwarded(t *testing.T) { SendFunc: func(c tbapi.Chattable) (tbapi.Message, error) { return tbapi.Message{Text: c.(tbapi.MessageConfig).Text, From: &tbapi.User{UserName: "user"}}, nil }, + RequestFunc: func(c tbapi.Chattable) (*tbapi.APIResponse, error) { + return &tbapi.APIResponse{Ok: true}, nil + }, } b := &mocks.BotMock{ OnMessageFunc: func(msg bot.Message) bot.Response { @@ -297,18 +303,22 @@ func TestTelegramListener_DoWithForwarded(t *testing.T) { AdminGroup: "123", StartupMsg: "startup", SuperUsers: SuperUser{"umputun"}, + Locator: NewLocator(10 * time.Minute), } ctx, cancel := context.WithTimeout(context.Background(), 500*time.Minute) defer cancel() + l.Locator.Add("text 123", 123, 88, 999999) // add message to locator + updMsg := tbapi.Update{ Message: &tbapi.Message{ Chat: &tbapi.Chat{ID: 123}, Text: "text 123", - From: &tbapi.User{UserName: "umputun"}, + From: &tbapi.User{UserName: "umputun", ID: 77}, Date: int(time.Date(2020, 2, 11, 19, 35, 55, 9, time.UTC).Unix()), ForwardSenderName: "forwarded_name", + MessageID: 999999, }, } @@ -324,6 +334,14 @@ func TestTelegramListener_DoWithForwarded(t *testing.T) { assert.Equal(t, "startup", mockAPI.SendCalls()[0].C.(tbapi.MessageConfig).Text) require.Equal(t, 1, len(b.UpdateSpamCalls())) assert.Equal(t, "text 123", b.UpdateSpamCalls()[0].Msg) + + assert.Equal(t, 2, len(mockAPI.RequestCalls())) + assert.Equal(t, int64(123), mockAPI.RequestCalls()[0].C.(tbapi.DeleteMessageConfig).ChatID) + assert.Equal(t, 999999, mockAPI.RequestCalls()[0].C.(tbapi.DeleteMessageConfig).MessageID) + + assert.Equal(t, int64(123), mockAPI.RequestCalls()[1].C.(tbapi.RestrictChatMemberConfig).ChatID) + assert.Equal(t, int64(88), mockAPI.RequestCalls()[1].C.(tbapi.RestrictChatMemberConfig).UserID) + } func TestTelegram_transformTextMessage(t *testing.T) { @@ -500,7 +518,7 @@ func TestTelegramListener_isChatAllowed(t *testing.T) { } } -func TestTelegramListener_forwardToAdmin(t *testing.T) { +func TestTelegramListener_reportToAdminChat(t *testing.T) { mockAPI := &mocks.TbAPIMock{ SendFunc: func(c tbapi.Chattable) (tbapi.Message, error) { return tbapi.Message{}, nil @@ -524,7 +542,7 @@ func TestTelegramListener_forwardToAdmin(t *testing.T) { Text: "Test\n\n_message_", } - listener.forwardToAdmin("testUser", msg) + listener.reportToAdminChat("testUser", msg) require.Equal(t, 1, len(mockAPI.SendCalls())) assert.Equal(t, int64(123), mockAPI.SendCalls()[0].C.(tbapi.MessageConfig).ChatID) @@ -582,3 +600,43 @@ func TestTelegramListener_isAdminChat(t *testing.T) { }) } } + +func TestSuperUser_IsSuper(t *testing.T) { + tests := []struct { + name string + super SuperUser + userName string + want bool + }{ + { + name: "User is a super user", + super: SuperUser{"Alice", "Bob"}, + userName: "Alice", + want: true, + }, + { + name: "User is not a super user", + super: SuperUser{"Alice", "Bob"}, + userName: "Charlie", + want: false, + }, + { + name: "User is a super user with slash prefix", + super: SuperUser{"/Alice", "Bob"}, + userName: "Alice", + want: true, + }, + { + name: "User is not a super user with slash prefix", + super: SuperUser{"/Alice", "Bob"}, + userName: "Charlie", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, tt.super.IsSuper(tt.userName)) + }) + } +} diff --git a/app/events/locator.go b/app/events/locator.go new file mode 100644 index 00000000..1be117c7 --- /dev/null +++ b/app/events/locator.go @@ -0,0 +1,76 @@ +package events + +import ( + "crypto/sha256" + "fmt" + "time" +) + +// Locator stores messages for a given time period. +// It is used to locate the message in the chat by its hash. +// Useful to match messages from admin chat (only text available) to the original message. +// Note: it is not thread-safe, use it from a single goroutine only. +type Locator struct { + ttl time.Duration // how long to keep messages + data map[string]MsgMeta // message hash -> message meta + lastRemoval time.Time // last time cleanup was performed + cleanupDuration time.Duration // how often to perform cleanup +} + +// MsgMeta stores message metadata +type MsgMeta struct { + time time.Time + chatID int64 + userID int64 + msgID int +} + +func (m MsgMeta) String() string { + return fmt.Sprintf("{chatID: %d, userID: %d, msgID: %d, time: %s}", m.chatID, m.userID, m.msgID, m.time.Format(time.RFC3339)) +} + +// NewLocator creates new Locator +func NewLocator(ttl time.Duration) *Locator { + return &Locator{ + ttl: ttl, + data: make(map[string]MsgMeta), + lastRemoval: time.Now(), + cleanupDuration: 5 * time.Minute, + } +} + +// Get returns message MsgMeta for given msg +// this allows to match messages from admin chat (only text available) to the original message +func (l *Locator) Get(msg string) (MsgMeta, bool) { + hash := l.MsgHash(msg) + res, ok := l.data[hash] + return res, ok +} + +// MsgHash returns sha256 hash of a message +func (l *Locator) MsgHash(msg string) string { + return fmt.Sprintf("%x", sha256.Sum256([]byte(msg))) +} + +// Add adds messages to the locator and removes old messages +// Note: removes old messages only once per cleanupDuration and only if a new message is added +func (l *Locator) Add(msg string, chatID, userID int64, msgID int) { + l.data[l.MsgHash(msg)] = MsgMeta{ + time: time.Now(), + chatID: chatID, + userID: userID, + msgID: msgID, + } + + if time.Since(l.lastRemoval) < l.cleanupDuration { + return + } + + // remove old messages + for k, v := range l.data { + if time.Since(v.time) > l.ttl { + delete(l.data, k) + } + } + l.lastRemoval = time.Now() +} diff --git a/app/events/locator_test.go b/app/events/locator_test.go new file mode 100644 index 00000000..56cefb08 --- /dev/null +++ b/app/events/locator_test.go @@ -0,0 +1,89 @@ +package events + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewLocator(t *testing.T) { + ttl := 10 * time.Minute + locator := NewLocator(ttl) + require.NotNil(t, locator) + + assert.Equal(t, ttl, locator.ttl) + assert.NotZero(t, locator.cleanupDuration) + assert.NotNil(t, locator.data) + assert.WithinDuration(t, time.Now(), locator.lastRemoval, time.Second) +} + +func TestGet(t *testing.T) { + locator := NewLocator(10 * time.Minute) + + // adding a message + msg := "test message" + chatID := int64(123) + userID := int64(456) + msgID := 7890 + locator.Add(msg, chatID, userID, msgID) + + // test retrieval of existing message + info, found := locator.Get("test message") + require.True(t, found) + assert.Equal(t, msgID, info.msgID) + assert.Equal(t, chatID, info.chatID) + assert.Equal(t, userID, info.userID) + + // test retrieval of non-existing message + _, found = locator.Get("no such message") // non-existing msgID + assert.False(t, found) +} + +func TestMsgHash(t *testing.T) { + locator := NewLocator(10 * time.Minute) + + t.Run("hash for empty message", func(t *testing.T) { + hash := locator.MsgHash("") + assert.Equal(t, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", hash) + }) + + t.Run("hash for non-empty message", func(t *testing.T) { + hash := locator.MsgHash("test message") + assert.Equal(t, "3f0a377ba0a4a460ecb616f6507ce0d8cfa3e704025d4fda3ed0c5ca05468728", hash) + }) + + t.Run("hash for different non-empty message", func(t *testing.T) { + hash := locator.MsgHash("test message blah") + assert.Equal(t, "21b7035e5ab5664eb7571b1f63d96951d5554a5465302b9cdd2e3de510eda6d8", hash) + }) +} + +func TestAddAndCleanup(t *testing.T) { + ttl := 2 * time.Second + cleanupDuration := 1 * time.Second + locator := NewLocator(ttl) + locator.cleanupDuration = cleanupDuration + + // Adding a message + msg := "test message" + chatID := int64(123) + userID := int64(456) + msgID := 7890 + locator.Add(msg, chatID, userID, msgID) + + hash := locator.MsgHash(msg) + meta, exists := locator.data[hash] + require.True(t, exists) + assert.Equal(t, chatID, meta.chatID) + assert.Equal(t, userID, meta.userID) + assert.Equal(t, msgID, meta.msgID) + + // wait for cleanup duration and add another message to trigger cleanup + time.Sleep(cleanupDuration + time.Second) + locator.Add("another message", 789, 555, 1011) + + _, existsAfterCleanup := locator.data[hash] + assert.False(t, existsAfterCleanup) +} diff --git a/app/events/superuser.go b/app/events/superuser.go deleted file mode 100644 index faff9674..00000000 --- a/app/events/superuser.go +++ /dev/null @@ -1,18 +0,0 @@ -package events - -import ( - "strings" -) - -// SuperUser for moderators -type SuperUser []string - -// IsSuper checks if username in the list of super users -func (s SuperUser) IsSuper(userName string) bool { - for _, super := range s { - if strings.EqualFold(userName, super) || strings.EqualFold("/"+userName, super) { - return true - } - } - return false -} diff --git a/app/events/superuser_test.go b/app/events/superuser_test.go deleted file mode 100644 index 80dd4152..00000000 --- a/app/events/superuser_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package events - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSuperUser_IsSuper(t *testing.T) { - tests := []struct { - name string - super SuperUser - userName string - want bool - }{ - { - name: "User is a super user", - super: SuperUser{"Alice", "Bob"}, - userName: "Alice", - want: true, - }, - { - name: "User is not a super user", - super: SuperUser{"Alice", "Bob"}, - userName: "Charlie", - want: false, - }, - { - name: "User is a super user with slash prefix", - super: SuperUser{"/Alice", "Bob"}, - userName: "Alice", - want: true, - }, - { - name: "User is not a super user with slash prefix", - super: SuperUser{"/Alice", "Bob"}, - userName: "Charlie", - want: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, tt.super.IsSuper(tt.userName)) - }) - } -} diff --git a/app/main.go b/app/main.go index fb059f58..fa541c46 100644 --- a/app/main.go +++ b/app/main.go @@ -25,6 +25,7 @@ import ( "github.com/umputun/tg-spam/app/bot" "github.com/umputun/tg-spam/app/events" "github.com/umputun/tg-spam/app/server" + "github.com/umputun/tg-spam/lib" ) var opts struct { @@ -42,7 +43,8 @@ var opts struct { Group string `long:"group" env:"GROUP" description:"admin group name/id"` } `group:"admin" namespace:"admin" env-namespace:"ADMIN"` - TestingIDs []int64 `long:"testing-id" env:"TESTING_ID" env-delim:"," description:"testing ids, allow bot to reply to them"` + TestingIDs []int64 `long:"testing-id" env:"TESTING_ID" env-delim:"," description:"testing ids, allow bot to reply to them"` + HistoryDuration time.Duration `long:"history-duration" env:"HISTORY_DURATION" default:"1h" description:"history duration"` Logger struct { Enabled bool `long:"enabled" env:"ENABLED" description:"enable spam rotated logs"` @@ -60,19 +62,19 @@ var opts struct { } `group:"cas" namespace:"cas" env-namespace:"CAS"` Files struct { - SamplesSpamFile string `long:"samples-spam" env:"SAMPLES_SPAM" default:"data/spam-samples.txt" description:"path to spam samples"` - SamplesHamFile string `long:"samples-ham" env:"SAMPLES_HAM" default:"data/ham-samples.txt" description:"path to ham samples"` - ExcludeTokenFile string `long:"exclude-tokens" env:"EXCLUDE_TOKENS" default:"data/exclude-tokens.txt" description:"path to exclude tokens file"` - StopWordsFile string `long:"stop-words" env:"STOP_WORDS" default:"data/stop-words.txt" description:"path to stop words file"` - DynamicSpamFile string `long:"dynamic-spam" env:"DYNAMIC_SPAM" default:"data/spam-dynamic.txt" description:"path to dynamic spam file"` - DynamicHamFile string `long:"dynamic-ham" env:"DYNAMIC_HAM" default:"data/ham-dynamic.txt" description:"path to dynamic ham file"` + SamplesSpamFile string `long:"samples-spam" env:"SAMPLES_SPAM" default:"data/spam-samples.txt" description:"spam samples"` + SamplesHamFile string `long:"samples-ham" env:"SAMPLES_HAM" default:"data/ham-samples.txt" description:"ham samples"` + ExcludeTokenFile string `long:"exclude-tokens" env:"EXCLUDE_TOKENS" default:"data/exclude-tokens.txt" description:"exclude tokens file"` + StopWordsFile string `long:"stop-words" env:"STOP_WORDS" default:"data/stop-words.txt" description:"stop words file"` + DynamicSpamFile string `long:"dynamic-spam" env:"DYNAMIC_SPAM" default:"data/spam-dynamic.txt" description:"dynamic spam file"` + DynamicHamFile string `long:"dynamic-ham" env:"DYNAMIC_HAM" default:"data/ham-dynamic.txt" description:"dynamic ham file"` + WatchInterval time.Duration `long:"watch-interval" env:"WATCH_INTERVAL" default:"5s" description:"watch interval"` } `group:"files" namespace:"files" env-namespace:"FILES"` SimilarityThreshold float64 `long:"similarity-threshold" env:"SIMILARITY_THRESHOLD" default:"0.5" description:"spam threshold"` - - MinMsgLen int `long:"min-msg-len" env:"MIN_MSG_LEN" default:"50" description:"min message length to check"` - MaxEmoji int `long:"max-emoji" env:"MAX_EMOJI" default:"2" description:"max emoji count in message"` - ParanoidMode bool `long:"paranoid" env:"PARANOID" description:"paranoid mode, check all messages"` + MinMsgLen int `long:"min-msg-len" env:"MIN_MSG_LEN" default:"50" description:"min message length to check"` + MaxEmoji int `long:"max-emoji" env:"MAX_EMOJI" default:"2" description:"max emoji count in message, -1 to disable check"` + ParanoidMode bool `long:"paranoid" env:"PARANOID" description:"paranoid mode, check all messages"` Message struct { Startup string `long:"startup" env:"STARTUP" default:"" description:"startup message"` @@ -124,43 +126,69 @@ func execute(ctx context.Context) error { log.Print("[WARN] dry mode, no actual bans") } + // make telegram bot tbAPI, err := tbapi.NewBotAPI(opts.Telegram.Token) if err != nil { return fmt.Errorf("can't make telegram bot, %w", err) } tbAPI.Debug = opts.TGDbg - spamBot, err := bot.NewSpamFilter(ctx, bot.SpamParams{ - SpamSamplesFile: opts.Files.SamplesSpamFile, - HamSamplesFile: opts.Files.SamplesHamFile, - SpamDynamicFile: opts.Files.DynamicSpamFile, - HamDynamicFile: opts.Files.DynamicHamFile, - ExcludedTokensFile: opts.Files.ExcludeTokenFile, - StopWordsFile: opts.Files.StopWordsFile, - SimilarityThreshold: opts.SimilarityThreshold, + // make spam detector + detectorConfig := lib.Config{ MaxAllowedEmoji: opts.MaxEmoji, MinMsgLen: opts.MinMsgLen, - HTTPClient: &http.Client{Timeout: 5 * time.Second}, + SimilarityThreshold: opts.SimilarityThreshold, CasAPI: opts.CAS.API, - SpamMsg: opts.Message.Spam, - SpamDryMsg: opts.Message.Dry, - ParanoidMode: opts.ParanoidMode, - Dry: opts.Dry, - }) - if err != nil { + HTTPClient: &http.Client{Timeout: opts.CAS.Timeout}, + FirstMessageOnly: !opts.ParanoidMode, + } + detector := lib.NewDetector(detectorConfig) + log.Printf("[DEBUG] detector config: %+v", detectorConfig) + + if opts.Files.DynamicSpamFile != "" { + detector.WithSpamUpdater(bot.NewSampleUpdater(opts.Files.DynamicSpamFile)) + log.Printf("[DEBUG] dynamic spam file: %s", opts.Files.DynamicSpamFile) + } + if opts.Files.DynamicHamFile != "" { + detector.WithHamUpdater(bot.NewSampleUpdater(opts.Files.DynamicHamFile)) + log.Printf("[DEBUG] dynamic ham file: %s", opts.Files.DynamicHamFile) + } + + // make spam bot + spamBotParams := bot.SpamConfig{ + SpamSamplesFile: opts.Files.SamplesSpamFile, + HamSamplesFile: opts.Files.SamplesHamFile, + SpamDynamicFile: opts.Files.DynamicSpamFile, + HamDynamicFile: opts.Files.DynamicHamFile, + ExcludedTokensFile: opts.Files.ExcludeTokenFile, + StopWordsFile: opts.Files.StopWordsFile, + WatchDelay: opts.Files.WatchInterval, + SpamMsg: opts.Message.Spam, + SpamDryMsg: opts.Message.Dry, + Dry: opts.Dry, + } + spamBot := bot.NewSpamFilter(ctx, detector, spamBotParams) + log.Printf("[DEBUG] spam bot config: %+v", spamBotParams) + + if err = spamBot.ReloadSamples(); err != nil { return fmt.Errorf("can't make spam bot, %w", err) } - web, err := server.NewSpamWeb(tbAPI, server.Params{ + // make web server + srvParams := server.Config{ + Version: revision, TgGroup: opts.Telegram.Group, URL: opts.Admin.URL, Secret: opts.Admin.Secret, ListenAddr: opts.Admin.Address, - }) + } + web, err := server.NewSpamWeb(tbAPI, srvParams) + log.Printf("[DEBUG] web params: %+v", srvParams) if err != nil { return fmt.Errorf("can't make spam rest, %w", err) } + // make spam logger loggerWr, err := makeSpamLogWriter() if err != nil { return fmt.Errorf("can't make spam log writer, %w", err) @@ -179,9 +207,14 @@ func execute(ctx context.Context) error { AdminGroup: opts.Admin.Group, TestingIDs: opts.TestingIDs, SpamWeb: web, + Locator: events.NewLocator(opts.HistoryDuration), Dry: opts.Dry, } + log.Printf("[DEBUG] telegram listener config: {group: %s, idle: %v, super: %v, admin: %s, testing: %v, no-reply: %v, dry: %v}", + tgListener.Group, tgListener.IdleDuration, tgListener.SuperUsers, tgListener.AdminGroup, + tgListener.TestingIDs, tgListener.NoSpamReply, tgListener.Dry) + // activate web server if configured if opts.Admin.URL != "" && opts.Admin.Secret != "" { go func() { if err := web.Run(ctx); err != nil { @@ -192,16 +225,20 @@ func execute(ctx context.Context) error { log.Print("[WARN] admin web server is disabled") } + // run telegram listener and event processor if err := tgListener.Do(ctx); err != nil { return fmt.Errorf("telegram listener failed, %w", err) } return nil } +// makeSpamLogger creates spam logger to keep reports about spam messages +// it writes json lines to the provided writer func makeSpamLogger(wr io.Writer) events.SpamLogger { return events.SpamLoggerFunc(func(msg *bot.Message, response *bot.Response) { - log.Printf("[INFO] spam detected from %v, response: %s", msg.From, response.Text) - log.Printf("[DEBUG] spam message: %q", msg.Text) + text := strings.ReplaceAll(response.Text, "\n", " ") + log.Printf("[INFO] spam detected from %v, response: %s", msg.From, text) + log.Printf("[DEBUG] spam message: %q", text) m := struct { TimeStamp string `json:"ts"` DisplayName string `json:"display_name"` @@ -213,7 +250,7 @@ func makeSpamLogger(wr io.Writer) events.SpamLogger { DisplayName: msg.From.DisplayName, UserName: msg.From.Username, UserID: msg.From.ID, - Text: strings.ReplaceAll(msg.Text, "\n", " "), + Text: text, } line, err := json.Marshal(&m) if err != nil { @@ -226,6 +263,8 @@ func makeSpamLogger(wr io.Writer) events.SpamLogger { }) } +// makeSpamLogWriter creates spam log writer to keep reports about spam messages +// it parses options and makes lumberjack logger with rotation func makeSpamLogWriter() (accessLog io.WriteCloser, err error) { if !opts.Logger.Enabled { return nopWriteCloser{io.Discard}, nil diff --git a/app/main_test.go b/app/main_test.go index fea2fa04..9244e628 100644 --- a/app/main_test.go +++ b/app/main_test.go @@ -30,7 +30,7 @@ func TestMakeSpamLogger(t *testing.T) { } response := &bot.Response{ - Text: "Test response", + Text: "Test message", } logger.Save(msg, response) @@ -99,7 +99,7 @@ func TestMakeSpamLogWriter(t *testing.T) { t.Run("disabled", func(t *testing.T) { opts.Logger.Enabled = false opts.Logger.FileName = "/tmp" - opts.Logger.MaxSize = "1f" + opts.Logger.MaxSize = "10M" opts.Logger.MaxBackups = 1 writer, err := makeSpamLogWriter() assert.NoError(t, err) diff --git a/app/server/server.go b/app/server/server.go index 8856483b..50493966 100644 --- a/app/server/server.go +++ b/app/server/server.go @@ -12,36 +12,43 @@ import ( "strconv" "time" + "github.com/didip/tollbooth/v7" + "github.com/didip/tollbooth_chi" + "github.com/go-chi/chi" + "github.com/go-chi/chi/v5/middleware" + "github.com/go-pkgz/lgr" + "github.com/go-pkgz/rest" tbapi "github.com/go-telegram-bot-api/telegram-bot-api/v5" ) //go:generate moq --out mocks/tb_api.go --pkg mocks --with-resets --skip-ensure . TbAPI -// SpamWeb is a REST API for ban/unban actions +// SpamWeb is a web server for ban/unban actions type SpamWeb struct { - Params + Config TbAPI TbAPI chatID int64 } -// Params defines REST API parameters -type Params struct { +// Config defines web server parameters +type Config struct { + Version string // version to show in /ping Secret string // secret key to sign url tokens URL string // root url ListenAddr string // listen address TgGroup string // telegram group name/id } -// TbAPI is an interface for telegram bot API, only a subset of methods used +// TbAPI is an interface for telegram bot API, only a subset of all methods used type TbAPI interface { Send(c tbapi.Chattable) (tbapi.Message, error) Request(c tbapi.Chattable) (*tbapi.APIResponse, error) GetChat(config tbapi.ChatInfoConfig) (tbapi.Chat, error) } -// NewSpamWeb creates new REST API server -func NewSpamWeb(tbAPI TbAPI, params Params) (*SpamWeb, error) { - res := SpamWeb{Params: params, TbAPI: tbAPI} +// NewSpamWeb creates new server +func NewSpamWeb(tbAPI TbAPI, params Config) (*SpamWeb, error) { + res := SpamWeb{Config: params, TbAPI: tbAPI} chatID, err := res.getChatID(params.TgGroup) if err != nil { return nil, fmt.Errorf("can't get chat ID for %s: %w", params.TgGroup, err) @@ -50,20 +57,17 @@ func NewSpamWeb(tbAPI TbAPI, params Params) (*SpamWeb, error) { return &res, nil } -// Run starts REST API server +// Run starts server and accepts requests to unban users from telegram func (s *SpamWeb) Run(ctx context.Context) error { + router := chi.NewRouter() + router.Use(middleware.RealIP, rest.Recoverer(lgr.Default()), middleware.GetHead) + router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second)) + router.Use(rest.AppInfo("tg-spam", "umputun", s.Version), rest.Ping) + router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil))) - mux := http.NewServeMux() - mux.HandleFunc("/ping", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Application", "tg-spam") - if _, err := w.Write([]byte("pong")); err != nil { - log.Printf("[WARN] failed to write response, %v", err) - } - }) - mux.HandleFunc("/unban", s.unbanHandler) - - srv := &http.Server{Addr: s.ListenAddr, Handler: mux, ReadTimeout: 5 * time.Second, WriteTimeout: 5 * time.Second} + router.Get("/unban", s.unbanHandler) + srv := &http.Server{Addr: s.ListenAddr, Handler: router, ReadTimeout: 5 * time.Second, WriteTimeout: 5 * time.Second} go func() { <-ctx.Done() if err := srv.Shutdown(ctx); err != nil { @@ -78,31 +82,67 @@ func (s *SpamWeb) Run(ctx context.Context) error { return nil } +type htmlResponse struct { + Title string + Message string + Background string + Foreground string + StatusCode int +} + // UnbanHandler handles unban requests, GET /unban?user=&token= func (s *SpamWeb) unbanHandler(w http.ResponseWriter, r *http.Request) { id := r.URL.Query().Get("user") token := r.URL.Query().Get("token") - userID, err := s.getChatID(id) + userID, err := strconv.ParseInt(id, 10, 64) if err != nil { log.Printf("[WARN] failed to get user ID for %q, %v", id, err) - s.sendHTML(w, fmt.Sprintf("failed to get user ID for %q: %v", id, err), "Error", "#ff6347", "#ffffff", http.StatusBadRequest) + resp := htmlResponse{ + Title: "Error", + Message: fmt.Sprintf("failed to get user ID for %q: %v", id, err), + Background: "#ff6347", + Foreground: "#ffffff", + StatusCode: http.StatusBadRequest, + } + s.sendHTML(w, resp) return } expToken := fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("%d::%s", userID, s.Secret)))) if len(token) != len(expToken) || subtle.ConstantTimeCompare([]byte(token), []byte(expToken)) != 1 { log.Printf("[WARN] invalid token for %q", id) - s.sendHTML(w, fmt.Sprintf("invalid token for %q", id), "Error", "#ff6347", "#ffffff", http.StatusForbidden) + resp := htmlResponse{ + Title: "Error", + Message: fmt.Sprintf("invalid token for %q", id), + Background: "#ff6347", + Foreground: "#ffffff", + StatusCode: http.StatusForbidden, + } + s.sendHTML(w, resp) return } log.Printf("[INFO] unban user %d", userID) _, err = s.TbAPI.Request(tbapi.UnbanChatMemberConfig{ChatMemberConfig: tbapi.ChatMemberConfig{UserID: userID, ChatID: s.chatID}}) if err != nil { log.Printf("[WARN] failed to unban %s, %v", id, err) - s.sendHTML(w, fmt.Sprintf("failed to unban %s: %v", id, err), "Error", "#ff6347", "#ffffff", http.StatusInternalServerError) + resp := htmlResponse{ + Title: "Error", + Message: fmt.Sprintf("failed to unban %s: %v", id, err), + Background: "#ff6347", + Foreground: "#ffffff", + StatusCode: http.StatusInternalServerError, + } + s.sendHTML(w, resp) return } - s.sendHTML(w, fmt.Sprintf("user %d unbanned", userID), "Success", "#90ee90", "#000000", http.StatusOK) + resp := htmlResponse{ + Title: "Success", + Message: fmt.Sprintf("user %d unbanned", userID), + Background: "#90ee90", + Foreground: "#000000", + StatusCode: http.StatusOK, + } + s.sendHTML(w, resp) } func (s *SpamWeb) getChatID(group string) (int64, error) { @@ -126,24 +166,12 @@ func (s *SpamWeb) UnbanURL(userID int64) string { return fmt.Sprintf("%s/unban?user=%d&token=%s", s.URL, userID, key) } -func (s *SpamWeb) sendHTML(w http.ResponseWriter, msg, title, background, foreground string, statusCode int) { - tmplParams := struct { - Title string - Message string - Background string - Foreground string - }{ - Title: title, - Message: msg, - Background: background, - Foreground: foreground, - } - +func (s *SpamWeb) sendHTML(w http.ResponseWriter, resp htmlResponse) { w.Header().Set("Content-Type", "text/html") - w.WriteHeader(statusCode) + w.WriteHeader(resp.StatusCode) htmlTmpl := template.Must(template.New("msg").Parse(msgTemplate)) - if err := htmlTmpl.Execute(w, tmplParams); err != nil { + if err := htmlTmpl.Execute(w, resp); err != nil { log.Printf("[WARN] failed to execute template, %v", err) return } diff --git a/app/server/server_test.go b/app/server/server_test.go index 0c7ed6de..02f34a71 100644 --- a/app/server/server_test.go +++ b/app/server/server_test.go @@ -2,6 +2,7 @@ package server import ( "context" + "errors" "io" "net/http" "testing" @@ -32,7 +33,7 @@ func TestSpamRest_UnbanURL(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - srv := SpamWeb{Params: Params{URL: tt.url, Secret: tt.secret}} + srv := SpamWeb{Config: Config{URL: tt.url, Secret: tt.secret}} res := srv.UnbanURL(123) assert.Equal(t, tt.want, res) }) @@ -45,17 +46,23 @@ func TestSpamRest_Run(t *testing.T) { mockAPI := &mocks.TbAPIMock{ GetChatFunc: func(config tbapi.ChatInfoConfig) (tbapi.Chat, error) { + if config.ChatConfig.SuperGroupUsername == "xxx" { + return tbapi.Chat{}, errors.New("not found") + } if config.ChatConfig.SuperGroupUsername == "@group" { return tbapi.Chat{ID: 10}, nil } return tbapi.Chat{ID: 123}, nil }, RequestFunc: func(c tbapi.Chattable) (*tbapi.APIResponse, error) { + if c.(tbapi.UnbanChatMemberConfig).UserID == 666 { + return nil, errors.New("failed") + } return &tbapi.APIResponse{}, nil }, } - srv, err := NewSpamWeb(mockAPI, Params{ + srv, err := NewSpamWeb(mockAPI, Config{ ListenAddr: ":9900", URL: "http://localhost:9090", Secret: "secret", @@ -74,6 +81,18 @@ func TestSpamRest_Run(t *testing.T) { time.Sleep(100 * time.Millisecond) // wait for server to start + t.Run("ping", func(t *testing.T) { + mockAPI.ResetCalls() + resp, err := http.Get("http://localhost:9900/ping") + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, "text/plain", resp.Header.Get("Content-Type")) + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.Equal(t, "pong", string(body)) + }) + t.Run("unban forbidden, wrong token", func(t *testing.T) { mockAPI.ResetCalls() req, err := http.NewRequest("GET", "http://localhost:9900/unban?user=123&token=ssss", http.NoBody) @@ -91,6 +110,43 @@ func TestSpamRest_Run(t *testing.T) { assert.Equal(t, "text/html", resp.Header.Get("Content-Type")) }) + t.Run("unban failed, bad id", func(t *testing.T) { + mockAPI.ResetCalls() + req, err := http.NewRequest("GET", + "http://localhost:9900/unban?user=xxx&token=71199ea8c011a49df546451e456ad10b0016566a53c4861bf849ec6b2ad2a0b7", http.NoBody) + require.NoError(t, err) + client := &http.Client{Timeout: 5 * time.Second} + resp, err := client.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusBadRequest, resp.StatusCode) + assert.Equal(t, 0, len(mockAPI.RequestCalls())) + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + t.Logf("body: %s", body) + assert.Contains(t, string(body), "Error") + assert.Equal(t, "text/html", resp.Header.Get("Content-Type")) + }) + + t.Run("unban failed, unban request failed", func(t *testing.T) { + mockAPI.ResetCalls() + req, err := http.NewRequest("GET", + "http://localhost:9900/unban?user=666&token=4eeb1bfa92a5c9418e8708953daaba267f86df63281da9480c53206d4cb2be32", http.NoBody) + require.NoError(t, err) + client := &http.Client{Timeout: 5 * time.Second} + resp, err := client.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusInternalServerError, resp.StatusCode) + assert.Equal(t, 1, len(mockAPI.RequestCalls())) + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + t.Logf("body: %s", body) + assert.Contains(t, string(body), "Error") + assert.Equal(t, "text/html", resp.Header.Get("Content-Type")) + + }) + t.Run("unban allowed, matched token", func(t *testing.T) { mockAPI.ResetCalls() req, err := http.NewRequest("GET", diff --git a/data/spam-samples.txt b/data/spam-samples.txt index ea62eef1..06377c09 100644 --- a/data/spam-samples.txt +++ b/data/spam-samples.txt @@ -103,3 +103,4 @@ Baлютa тpeбуeт 18 лeт, жeлaния рaзвивaтьcя, и пyнктya Рaccкaжитe o ceбe вмecтe c Knyaz Service. Яpкo, нeoбычнo и интepecнo: Видeo-бaннepы, видeo-aвaтapы, лoгoтипы, дизaйн пocтoв и cтopиc — вcё для coздaния пoзитивнoгo имиджa. Офopмитe зaкaз oт 1000 pyб. Гapaнтии и бeзoпacнocть oплaты! Пpимepы paбoт и ycлoвия - в лс Приветствую всех, есть уже освоенные мною курсы по Cryptocurrency. Если есть желание научиться пользоваться Биржами, понять как работает это направление и как можно себя в нем реализовать. Могу поделиться материалами за Спасибо📈💰 📈Beдeтся нaбop людeй нa yдaлeннyю paбoтy в сфepe apбитpaж kриптoвaлют! ✅ Bpeмя нa pa6oтy: 1-4 чaca в дeнь ✔️Пpeдocтaвляeм oбyчeниe 👍 Гapaнтиpoвaннaя пpибыль yжe нa этaпe oбyчeния 📌 C oбмeннuкaми или стopoнними сaйтaми нe рaбoтaeм, тoлькo бupжи-гигaнты ⚙️Зaинтepecoвaл? Ждy сooбщeния! +🔞 Мои фоточки без труcиков ➡️ @Alina_Ckls \ No newline at end of file diff --git a/go.mod b/go.mod index eac0fd01..b8ddbcd0 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,14 @@ module github.com/umputun/tg-spam go 1.21 require ( + github.com/didip/tollbooth/v7 v7.0.1 + github.com/didip/tollbooth_chi v0.0.0-20220719025231-d662a7f6928f github.com/fatih/color v1.16.0 github.com/fsnotify/fsnotify v1.7.0 + github.com/go-chi/chi v1.5.5 + github.com/go-chi/chi/v5 v5.0.10 github.com/go-pkgz/lgr v0.11.1 - github.com/go-pkgz/notify v1.0.0 + github.com/go-pkgz/rest v1.18.2 github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1 github.com/hashicorp/go-multierror v1.1.1 github.com/stretchr/testify v1.8.4 @@ -15,19 +19,12 @@ require ( ) require ( - github.com/aymerick/douceur v0.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/go-pkgz/email v0.4.1 // indirect - github.com/go-pkgz/repeater v1.1.3 // indirect - github.com/gorilla/css v1.0.1 // indirect - github.com/gorilla/websocket v1.5.1 // indirect + github.com/go-pkgz/expirable-cache v0.1.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/microcosm-cc/bluemonday v1.0.26 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/slack-go/slack v0.12.3 // indirect - golang.org/x/net v0.19.0 // indirect golang.org/x/sys v0.15.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 3872e856..8dfc8e6d 100644 --- a/go.sum +++ b/go.sum @@ -1,33 +1,27 @@ -github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= -github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/didip/tollbooth/v7 v7.0.0/go.mod h1:VZhDSGl5bDSPj4wPsih3PFa4Uh9Ghv8hgacaTm5PRT4= +github.com/didip/tollbooth/v7 v7.0.1 h1:TkT4sBKoQoHQFPf7blQ54iHrZiTDnr8TceU+MulVAog= +github.com/didip/tollbooth/v7 v7.0.1/go.mod h1:VZhDSGl5bDSPj4wPsih3PFa4Uh9Ghv8hgacaTm5PRT4= +github.com/didip/tollbooth_chi v0.0.0-20220719025231-d662a7f6928f h1:jtKwihcLmUC9BAhoJ9adCUqdSSZcOdH2KL7mPTUm2aw= +github.com/didip/tollbooth_chi v0.0.0-20220719025231-d662a7f6928f/go.mod h1:q9C80dnsuVRP2dAskjnXRNWdUJqtGgwG9wNrzt0019s= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-chi/chi v1.5.5 h1:vOB/HbEMt9QqBqErz07QehcOKHaWFtuj87tTDVz2qXE= +github.com/go-chi/chi v1.5.5/go.mod h1:C9JqLr3tIYjDOZpzn+BCuxY8z8vmca43EeMgyZt7irw= github.com/go-chi/chi/v5 v5.0.10 h1:rLz5avzKpjqxrYwXNfmjkrYYXOyLJd37pz53UFHC6vk= github.com/go-chi/chi/v5 v5.0.10/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= -github.com/go-pkgz/email v0.4.1 h1:2vtP2gibsSzqhz6eD5DklSp11m657XEVf17fuXaxMvk= -github.com/go-pkgz/email v0.4.1/go.mod h1:BdxglsQnymzhfdbnncEE72a6DrucZHy6I+42LK2jLEc= +github.com/go-pkgz/expirable-cache v0.1.0 h1:3bw0m8vlTK8qlwz5KXuygNBTkiKRTPrAGXU0Ej2AC1g= +github.com/go-pkgz/expirable-cache v0.1.0/go.mod h1:GTrEl0X+q0mPNqN6dtcQXksACnzCBQ5k/k1SwXJsZKs= github.com/go-pkgz/lgr v0.11.1 h1:hXFhZcznehI6imLhEa379oMOKFz7TQUmisAqb3oLOSM= github.com/go-pkgz/lgr v0.11.1/go.mod h1:tgDF4RXQnBfIgJqjgkv0yOeTQ3F1yewWIZkpUhHnAkU= -github.com/go-pkgz/notify v1.0.0 h1:bCfB04bv+TK20tOz/eQ2ihZe286upHAnQw8mF6Wa/eI= -github.com/go-pkgz/notify v1.0.0/go.mod h1:mEN+rqyDhOuLiSXpz1iVNMCYd3sZ1SnhVbEMzwO4N6I= -github.com/go-pkgz/repeater v1.1.3 h1:q6+JQF14ESSy28Dd7F+wRelY4F+41HJ0LEy/szNnMiE= -github.com/go-pkgz/repeater v1.1.3/go.mod h1:hVTavuO5x3Gxnu8zW7d6sQBfAneKV8X2FjU48kGfpKw= +github.com/go-pkgz/rest v1.18.2 h1:eJYj1qlLJvTx86R4o+XmlKHOAGAX42WeG9PZrJud/e0= +github.com/go-pkgz/rest v1.18.2/go.mod h1:Po+W6zQzpMPP6XDGLdAN2aW7UKk1IyrLSb48Lp1N3oQ= github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1 h1:wG8n/XJQ07TmjbITcGiUaOtXxdrINDz1b0J1w0SzqDc= github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1/go.mod h1:A2S0CWkNylc2phvKXWBBdD3K0iGnDBGbzRpISP2zBl8= -github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho= -github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= -github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= -github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -38,29 +32,22 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58= -github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/slack-go/slack v0.12.3 h1:92/dfFU8Q5XP6Wp5rr5/T5JHLM5c5Smtn53fhToAP88= -github.com/slack-go/slack v0.12.3/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/umputun/go-flags v1.5.1 h1:vRauoXV3Ultt1HrxivSxowbintgZLJE+EcBy5ta3/mY= github.com/umputun/go-flags v1.5.1/go.mod h1:nTbvsO/hKqe7Utri/NoyN18GR3+EWf+9RrmsdwdhrEc= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/app/bot/classifier.go b/lib/classifier.go similarity index 99% rename from app/bot/classifier.go rename to lib/classifier.go index 9472c320..9a74e4cc 100644 --- a/app/bot/classifier.go +++ b/lib/classifier.go @@ -1,10 +1,8 @@ -package bot +package lib -// based on the code from https://github.com/RadhiFadlillah/go-bayesian/blob/master/classifier.go +import "math" -import ( - "math" -) +// based on the code from https://github.com/RadhiFadlillah/go-bayesian/blob/master/classifier.go // Class is alias of string, representing class of a document type Class string diff --git a/app/bot/classifier_test.go b/lib/classifier_test.go similarity index 99% rename from app/bot/classifier_test.go rename to lib/classifier_test.go index 6dd6cc97..7f4e6fbf 100644 --- a/app/bot/classifier_test.go +++ b/lib/classifier_test.go @@ -1,4 +1,4 @@ -package bot +package lib import ( "testing" diff --git a/lib/detector.go b/lib/detector.go new file mode 100644 index 00000000..0d3e0a93 --- /dev/null +++ b/lib/detector.go @@ -0,0 +1,433 @@ +package lib + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "log" + "math" + "net/http" + "strings" + "sync" +) + +//go:generate moq --out mocks/sample_updater.go --pkg mocks --skip-ensure . SampleUpdater +//go:generate moq --out mocks/http_client.go --pkg mocks --skip-ensure . HTTPClient + +// Detector is a spam detector, thread-safe. +type Detector struct { + Config + classifier Classifier + tokenizedSpam []map[string]int + approvedUsers map[int64]bool + stopWords []string + excludedTokens []string + + spamSamplesUpd SampleUpdater + hamSamplesUpd SampleUpdater + + lock sync.RWMutex +} + +// Config is a set of parameters for Detector. +type Config struct { + SimilarityThreshold float64 // threshold for spam similarity, 0.0 - 1.0 + MinMsgLen int // minimum message length to check + MaxAllowedEmoji int // maximum number of emojis allowed in a message + CasAPI string // CAS API URL + FirstMessageOnly bool // if true, only the first message from a user is checked + HTTPClient HTTPClient // http client to use for requests +} + +// CheckResult is a result of spam check. +type CheckResult struct { + Name string // name of the check + Spam bool // true if spam + Details string // details of the check +} + +// LoadResult is a result of loading samples. +type LoadResult struct { + ExcludedTokens int // number of excluded tokens + SpamSamples int // number of spam samples + HamSamples int // number of ham samples + StopWords int // number of stop words (phrases) +} + +// SampleUpdater is an interface for updating spam/ham samples on the fly. +type SampleUpdater interface { + Append(msg string) error // append a message to the samples storage + Reader() (io.ReadCloser, error) // return a reader for the samples storage +} + +// HTTPClient wrap http.Client to allow mocking +type HTTPClient interface { + Do(req *http.Request) (*http.Response, error) +} + +// NewDetector makes a new Detector with the given config. +func NewDetector(p Config) *Detector { + return &Detector{ + Config: p, + classifier: NewClassifier(), + approvedUsers: make(map[int64]bool), + tokenizedSpam: []map[string]int{}, + } +} + +// Check checks if a given message is spam. Returns true if spam. +// Also returns a list of check results. +func (d *Detector) Check(msg string, userID int64) (spam bool, cr []CheckResult) { + + if len([]rune(msg)) < d.MinMsgLen { + return false, []CheckResult{{Name: "message length", Spam: false, Details: "too short"}} + } + + d.lock.RLock() + defer d.lock.RUnlock() + + if d.FirstMessageOnly && d.approvedUsers[userID] { + return false, []CheckResult{{Name: "pre-approved", Spam: false, Details: "user already approved"}} + } + + if len(d.stopWords) > 0 { + cr = append(cr, d.isStopWord(msg)) + } + + if d.MaxAllowedEmoji >= 0 { + cr = append(cr, d.isManyEmojis(msg)) + } + + if d.SimilarityThreshold > 0 && len(d.tokenizedSpam) > 0 { + cr = append(cr, d.isSpamSimilarityHigh(msg)) + } + + if d.classifier.NAllDocument > 0 { + cr = append(cr, d.isSpamClassified(msg)) + } + + if d.CasAPI != "" { + cr = append(cr, d.isCasSpam(userID)) + } + + for _, r := range cr { + if r.Spam { + return true, cr + } + } + + if d.FirstMessageOnly { + d.approvedUsers[userID] = true + } + + return false, cr +} + +// Reset resets spam samples/classifier, excluded tokens, stop words and approved users. +func (d *Detector) Reset() { + d.lock.Lock() + defer d.lock.Unlock() + + d.tokenizedSpam = []map[string]int{} + d.excludedTokens = []string{} + d.classifier.Reset() + d.approvedUsers = make(map[int64]bool) + d.stopWords = []string{} +} + +// WithSpamUpdater sets a SampleUpdater for spam samples. +func (d *Detector) WithSpamUpdater(s SampleUpdater) { + d.spamSamplesUpd = s +} + +// WithHamUpdater sets a SampleUpdater for ham samples. +func (d *Detector) WithHamUpdater(s SampleUpdater) { + d.hamSamplesUpd = s +} + +// LoadSamples loads spam samples from a reader and updates the classifier. +// Reset spam, ham samples/classifier, and excluded tokens. +func (d *Detector) LoadSamples(exclReader io.Reader, spamReaders, hamReaders []io.Reader) (LoadResult, error) { + d.lock.Lock() + defer d.lock.Unlock() + + d.tokenizedSpam = []map[string]int{} + d.excludedTokens = []string{} + d.classifier.Reset() + + // excluded tokens should be loaded before spam samples + for t := range d.tokenChan(exclReader) { + d.excludedTokens = append(d.excludedTokens, strings.ToLower(t)) + } + lr := LoadResult{ExcludedTokens: len(d.excludedTokens)} + + // load spam samples and update the classifier with them + docs := []Document{} + for token := range d.tokenChan(spamReaders...) { + tokenizedSpam := d.tokenize(token) + d.tokenizedSpam = append(d.tokenizedSpam, tokenizedSpam) // add to list of samples + tokens := make([]string, 0, len(tokenizedSpam)) + for token := range tokenizedSpam { + tokens = append(tokens, token) + } + docs = append(docs, Document{Class: "spam", Tokens: tokens}) + lr.SpamSamples++ + } + + for token := range d.tokenChan(hamReaders...) { + tokenizedSpam := d.tokenize(token) + tokens := make([]string, 0, len(tokenizedSpam)) + for token := range tokenizedSpam { + tokens = append(tokens, token) + } + docs = append(docs, Document{Class: "ham", Tokens: tokens}) + lr.HamSamples++ + } + + d.classifier.Learn(docs...) + + return lr, nil +} + +// LoadStopWords loads stop words from a reader. Reset stop words list before loading. +func (d *Detector) LoadStopWords(readers ...io.Reader) (LoadResult, error) { + d.lock.Lock() + defer d.lock.Unlock() + d.stopWords = []string{} + for t := range d.tokenChan(readers...) { + d.stopWords = append(d.stopWords, strings.ToLower(t)) + } + log.Printf("[INFO] loaded %d stop words", len(d.stopWords)) + return LoadResult{StopWords: len(d.stopWords)}, nil +} + +// UpdateSpam appends a message to the spam samples file and updates the classifier +// doesn't reset state, update append spam samples +func (d *Detector) UpdateSpam(msg string) error { + d.lock.Lock() + defer d.lock.Unlock() + + if d.spamSamplesUpd == nil { + return nil + } + + // write to dynamic samples storage + if err := d.spamSamplesUpd.Append(msg); err != nil { + return fmt.Errorf("can't update spam samples: %w", err) + } + + // load spam samples and update the classifier with them + docs := []Document{} + for token := range d.tokenChan(bytes.NewBufferString(msg)) { + tokenizedSpam := d.tokenize(token) + d.tokenizedSpam = append(d.tokenizedSpam, tokenizedSpam) // add to list of samples + tokens := make([]string, 0, len(tokenizedSpam)) + for token := range tokenizedSpam { + tokens = append(tokens, token) + } + docs = append(docs, Document{Class: "spam", Tokens: tokens}) + } + d.classifier.Learn(docs...) + return nil +} + +// UpdateHam appends a message to the ham samples file and updates the classifier +// doesn't reset state, update append ham samples +func (d *Detector) UpdateHam(msg string) error { + d.lock.Lock() + defer d.lock.Unlock() + + if d.hamSamplesUpd == nil { + return nil + } + + if err := d.hamSamplesUpd.Append(msg); err != nil { + return fmt.Errorf("can't update ham samples: %w", err) + } + + // load ham samples and update the classifier with them + docs := []Document{} + for token := range d.tokenChan(bytes.NewBufferString(msg)) { + tokenizedHam := d.tokenize(token) + tokens := make([]string, 0, len(tokenizedHam)) + for token := range tokenizedHam { + tokens = append(tokens, token) + } + docs = append(docs, Document{Class: "ham", Tokens: tokens}) + } + d.classifier.Learn(docs...) + return nil +} + +// tokenChan parses readers and returns a channel of tokens. +// A line per-token or comma-separated "tokens" supported +func (d *Detector) tokenChan(readers ...io.Reader) <-chan string { + resCh := make(chan string) + + go func() { + defer close(resCh) + + for _, reader := range readers { + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + line := scanner.Text() + if strings.Contains(line, ",") && strings.HasPrefix(line, "\"") { + // line with comma-separated tokens + lineTokens := strings.Split(line, ",") + for _, token := range lineTokens { + cleanToken := strings.Trim(token, " \"\n\r\t") + if cleanToken != "" { + resCh <- cleanToken + } + } + continue + } + // each line with a single token + cleanToken := strings.Trim(line, " \n\r\t") + if cleanToken != "" { + resCh <- cleanToken + } + } + + if err := scanner.Err(); err != nil { + log.Printf("[WARN] failed to read tokens, error=%v", err) + } + } + }() + + return resCh +} + +// tokenize takes a string and returns a map where the keys are unique words (tokens) +// and the values are the frequencies of those words in the string. +// exclude tokens representing common words. +func (d *Detector) tokenize(inp string) map[string]int { + isExcludedToken := func(token string) bool { + for _, w := range d.excludedTokens { + if strings.EqualFold(token, w) { + return true + } + } + return false + } + + tokenFrequency := make(map[string]int) + tokens := strings.Fields(inp) + for _, token := range tokens { + if isExcludedToken(token) { + continue + } + token = cleanEmoji(token) + token = strings.Trim(token, ".,!?-:;()#") + token = strings.ToLower(token) + if len([]rune(token)) < 3 { + continue + } + tokenFrequency[strings.ToLower(token)]++ + } + return tokenFrequency +} + +// isSpam checks if a given message is similar to any of the known bad messages. +func (d *Detector) isSpamSimilarityHigh(msg string) CheckResult { + // check for spam similarity + tokenizedMessage := d.tokenize(msg) + maxSimilarity := 0.0 + for _, spam := range d.tokenizedSpam { + similarity := d.cosineSimilarity(tokenizedMessage, spam) + if similarity > maxSimilarity { + maxSimilarity = similarity + } + if similarity >= d.SimilarityThreshold { + return CheckResult{Spam: true, Name: "similarity", + Details: fmt.Sprintf("%0.2f/%0.2f", maxSimilarity, d.SimilarityThreshold)} + } + } + return CheckResult{Spam: false, Name: "similarity", Details: fmt.Sprintf("%0.2f/%0.2f", maxSimilarity, d.SimilarityThreshold)} +} + +// cosineSimilarity calculates the cosine similarity between two token frequency maps. +func (d *Detector) cosineSimilarity(a, b map[string]int) float64 { + if len(a) == 0 || len(b) == 0 { + return 0.0 + } + + dotProduct := 0 // sum of product of corresponding frequencies + normA, normB := 0, 0 // square root of sum of squares of frequencies + + for key, val := range a { + dotProduct += val * b[key] + normA += val * val + } + for _, val := range b { + normB += val * val + } + + if normA == 0 || normB == 0 { + return 0.0 + } + + // cosine similarity formula + return float64(dotProduct) / (math.Sqrt(float64(normA)) * math.Sqrt(float64(normB))) +} + +func (d *Detector) isCasSpam(msgID int64) CheckResult { + reqURL := fmt.Sprintf("%s/check?user_id=%d", d.CasAPI, msgID) + req, err := http.NewRequest("GET", reqURL, http.NoBody) + if err != nil { + return CheckResult{Spam: false, Name: "cas", Details: fmt.Sprintf("failed to make request %s: %v", reqURL, err)} + } + + resp, err := d.HTTPClient.Do(req) + if err != nil { + return CheckResult{Spam: false, Name: "cas", Details: fmt.Sprintf("ffailed to send request %s: %v", reqURL, err)} + } + defer resp.Body.Close() + + respData := struct { + OK bool `json:"ok"` // ok means user is a spammer + Description string `json:"description"` + }{} + + if err := json.NewDecoder(resp.Body).Decode(&respData); err != nil { + return CheckResult{Spam: false, Name: "cas", Details: fmt.Sprintf("failed to parse response from %s: %v", reqURL, err)} + } + + if respData.OK { + return CheckResult{Name: "cas", Spam: true, Details: respData.Description} + } + details := respData.Description + if details == "" { + details = "not found" + } + return CheckResult{Name: "cas", Spam: false, Details: details} +} + +func (d *Detector) isSpamClassified(msg string) CheckResult { + // Classify tokens from a document + tm := d.tokenize(msg) + tokens := make([]string, 0, len(tm)) + for token := range tm { + tokens = append(tokens, token) + } + allScores, class, certain := d.classifier.Classify(tokens...) + return CheckResult{Name: "classifier", Spam: class == "spam" && certain, + Details: fmt.Sprintf("spam:%.4f, ham:%.4f", allScores["spam"], allScores["ham"])} +} + +func (d *Detector) isStopWord(msg string) CheckResult { + cleanMsg := cleanEmoji(strings.ToLower(msg)) + for _, word := range d.stopWords { + if strings.Contains(cleanMsg, strings.ToLower(word)) { + return CheckResult{Name: "stopword", Spam: true, Details: word} + } + } + return CheckResult{Name: "stopword", Spam: false} +} + +func (d *Detector) isManyEmojis(msg string) CheckResult { + count := countEmoji(msg) + return CheckResult{Name: "emoji", Spam: count > d.MaxAllowedEmoji, Details: fmt.Sprintf("%d/%d", count, d.MaxAllowedEmoji)} +} diff --git a/lib/detector_test.go b/lib/detector_test.go new file mode 100644 index 00000000..b0eae538 --- /dev/null +++ b/lib/detector_test.go @@ -0,0 +1,411 @@ +package lib + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/umputun/tg-spam/lib/mocks" +) + +func TestDetector_tokenize(t *testing.T) { + tests := []struct { + name string + input string + expected map[string]int + }{ + {name: "empty", input: "", expected: map[string]int{}}, + {name: "no filters or cleanups", input: "hello world", expected: map[string]int{"hello": 1, "world": 1}}, + {name: "with excluded tokens", input: "hello world the she", expected: map[string]int{"hello": 1, "world": 1}}, + {name: "with short tokens", input: "hello world the she a or", expected: map[string]int{"hello": 1, "world": 1}}, + {name: "with repeated tokens", input: "hello world hello world", expected: map[string]int{"hello": 2, "world": 2}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := Detector{ + excludedTokens: []string{"the", "she"}, + } + assert.Equal(t, tt.expected, d.tokenize(tt.input)) + }) + } +} + +func TestDetector_tokenChan(t *testing.T) { + tests := []struct { + name string + input string + expected []string + }{ + {name: "empty", input: "", expected: []string{}}, + {name: "token per line", input: "hello\nworld", expected: []string{"hello", "world"}}, + {name: "token per line", input: "hello 123\nworld", expected: []string{"hello 123", "world"}}, + {name: "token per line with spaces", input: "hello \n world", expected: []string{"hello", "world"}}, + {name: "tokens comma separated", input: "\"hello\",\"world\"\nsomething", expected: []string{"hello", "world", "something"}}, + {name: "tokens comma separated, extra EOL", input: "\"hello\",world\nsomething\n", expected: []string{"hello", "world", "something"}}, + {name: "tokens comma separated, empty tokens", input: "\"hello\",world,\"\"\nsomething\n ", expected: []string{"hello", "world", "something"}}, + } + + d := Detector{} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ch := d.tokenChan(bytes.NewBufferString(tt.input)) + res := []string{} + for token := range ch { + res = append(res, token) + } + assert.Equal(t, tt.expected, res) + }) + } +} + +func TestDetector_tokenChanMultipleReaders(t *testing.T) { + d := Detector{} + ch := d.tokenChan(bytes.NewBufferString("hello\nworld"), bytes.NewBufferString("something, new")) + res := []string{} + for token := range ch { + res = append(res, token) + } + assert.Equal(t, []string{"hello", "world", "something, new"}, res) +} + +func TestDetector_CheckStopWords(t *testing.T) { + d := NewDetector(Config{MaxAllowedEmoji: -1}) + lr, err := d.LoadStopWords(bytes.NewBufferString("в личку\nвсем привет")) + require.NoError(t, err) + assert.Equal(t, LoadResult{StopWords: 2}, lr) + + tests := []struct { + name string + message string + expected bool + }{ + { + name: "Stop word present", + message: "Hello, please send me a message в личку", + expected: true, + }, + { + name: "Stop word present with emoji", + message: "👋Всем привет\nИщу амбициозного человека к се6е в команду\nКто в поисках дополнительного заработка или хочет попробовать себя в новой сфере деятельности! 👨🏻\u200d💻\nПишите в лс✍️", + expected: true, + }, + { + name: "No stop word present", + message: "Hello, how are you?", + expected: false, + }, + { + name: "Case insensitive stop word present", + message: "Hello, please send me a message В ЛИЧКУ", + expected: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + spam, cr := d.Check(test.message, 0) + assert.Equal(t, test.expected, spam) + require.Len(t, cr, 1) + assert.Equal(t, "stopword", cr[0].Name) + t.Logf("%+v", cr[0].Details) + if test.expected { + assert.Subset(t, d.stopWords, []string{cr[0].Details}) + } + }) + } +} + +//nolint:stylecheck // it has unicode symbols purposely +func TestDetector_CheckEmojis(t *testing.T) { + d := NewDetector(Config{MaxAllowedEmoji: 2}) + tests := []struct { + name string + input string + count int + spam bool + }{ + {"NoEmoji", "Hello, world!", 0, false}, + {"OneEmoji", "Hi there 👋", 1, false}, + {"TwoEmojis", "Good morning 🌞🌻", 2, false}, + {"Mixed", "👨‍👩‍👧‍👦 Family emoji", 1, false}, + {"EmojiSequences", "🏳️‍🌈 Rainbow flag", 1, false}, + {"TextAfterEmoji", "😊 Have a nice day!", 1, false}, + {"OnlyEmojis", "😁🐶🍕", 3, true}, + {"WithCyrillic", "Привет 🌞 🍕 мир! 👋", 3, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spam, cr := d.Check(tt.input, 0) + assert.Equal(t, tt.spam, spam) + require.Len(t, cr, 1) + assert.Equal(t, "emoji", cr[0].Name) + assert.Equal(t, tt.spam, cr[0].Spam) + assert.Equal(t, fmt.Sprintf("%d/2", tt.count), cr[0].Details) + }) + } +} + +func TestSpam_CheckIsCasSpam(t *testing.T) { + tests := []struct { + name string + mockResp string + mockStatusCode int + expected bool + }{ + { + name: "User is not a spammer", + mockResp: `{"ok": false, "description": "Not a spammer"}`, + mockStatusCode: 200, + expected: false, + }, + { + name: "User is a spammer", + mockResp: `{"ok": true, "description": "Is a spammer"}`, + mockStatusCode: 200, + expected: true, + }, + { + name: "HTTP error", + mockResp: "{}", + mockStatusCode: 500, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockedHTTPClient := &mocks.HTTPClientMock{ + DoFunc: func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: tt.mockStatusCode, + Body: io.NopCloser(bytes.NewBufferString(tt.mockResp)), + }, nil + }, + } + + d := NewDetector(Config{ + CasAPI: "http://localhost", + HTTPClient: mockedHTTPClient, + MaxAllowedEmoji: -1, + FirstMessageOnly: true, + }) + spam, cr := d.Check("", 123) + assert.Equal(t, tt.expected, spam) + require.Len(t, cr, 1) + assert.Equal(t, "cas", cr[0].Name) + assert.Equal(t, tt.expected, cr[0].Spam) + + respDetails := struct { + OK bool `json:"ok"` + Description string `json:"description"` + }{} + err := json.Unmarshal([]byte(tt.mockResp), &respDetails) + require.NoError(t, err) + assert.Equal(t, respDetails.Description, respDetails.Description) + assert.Equal(t, 1, len(mockedHTTPClient.DoCalls())) + }) + } +} + +func TestDetector_CheckSimilarity(t *testing.T) { + d := NewDetector(Config{MaxAllowedEmoji: -1}) + spamSamples := strings.NewReader("win free iPhone\nlottery prize xyz") + lr, err := d.LoadSamples(strings.NewReader("xyz"), []io.Reader{spamSamples}, nil) + require.NoError(t, err) + assert.Equal(t, LoadResult{ExcludedTokens: 1, SpamSamples: 2}, lr) + d.classifier.Reset() // we don't need a classifier for this test + assert.Len(t, d.tokenizedSpam, 2) + t.Logf("%+v", d.tokenizedSpam) + assert.Equal(t, map[string]int{"win": 1, "free": 1, "iphone": 1}, d.tokenizedSpam[0]) + assert.Equal(t, map[string]int{"lottery": 1, "prize": 1}, d.tokenizedSpam[1]) + + tests := []struct { + name string + message string + threshold float64 + expected bool + }{ + {"Not Spam", "Hello, how are you?", 0.5, false}, + {"Exact Match", "Win a free iPhone now!", 0.5, true}, + {"Similar Match", "You won a lottery prize!", 0.3, true}, + {"High Threshold", "You won a lottery prize!", 0.9, false}, + {"Partial Match", "win free", 0.9, false}, + {"Low Threshold", "win free", 0.8, true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + d.Config.SimilarityThreshold = test.threshold // Update threshold for each test case + spam, cr := d.Check(test.message, 0) + assert.Equal(t, test.expected, spam) + require.Len(t, cr, 1) + assert.Equal(t, "similarity", cr[0].Name) + }) + } +} + +func TestDetector_CheckClassificator(t *testing.T) { + d := NewDetector(Config{MaxAllowedEmoji: -1}) + spamSamples := strings.NewReader("win free iPhone\nlottery prize xyz") + hamsSamples := strings.NewReader("hello world\nhow are you\nhave a good day") + lr, err := d.LoadSamples(strings.NewReader("xyz"), []io.Reader{spamSamples}, []io.Reader{hamsSamples}) + require.NoError(t, err) + assert.Equal(t, LoadResult{ExcludedTokens: 1, SpamSamples: 2, HamSamples: 3}, lr) + d.tokenizedSpam = nil // we don't need tokenizedSpam samples for this test + assert.Equal(t, 5, d.classifier.NAllDocument) + exp := map[string]map[Class]int{"win": {"spam": 1}, "free": {"spam": 1}, "iphone": {"spam": 1}, "lottery": {"spam": 1}, + "prize": {"spam": 1}, "hello": {"ham": 1}, "world": {"ham": 1}, "how": {"ham": 1}, "are": {"ham": 1}, "you": {"ham": 1}, + "have": {"ham": 1}, "good": {"ham": 1}, "day": {"ham": 1}} + assert.Equal(t, exp, d.classifier.LearningResults) + + tests := []struct { + name string + message string + expected bool + desc string + }{ + {"clean ham", "Hello, how are you?", false, "spam:-12.4778, ham:-9.9163"}, + {"clean spam", "Win a free iPhone now!", true, "spam:-10.3983, ham:-12.6889"}, + {"mostly spam", "You won a free lottery iphone, have a good day", true, "spam:-21.9598, ham:-22.0944"}, + {"mostly ham", "win a good day", false, "spam:-8.8943, ham:-8.2581"}, + {"a little bit spam", "free blah another one user writes good things iPhone day", true, "spam:-28.4337, ham:-29.5698"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + spam, cr := d.Check(test.message, 0) + assert.Equal(t, test.expected, spam) + require.Len(t, cr, 1) + assert.Equal(t, "classifier", cr[0].Name) + assert.Equal(t, test.expected, cr[0].Spam) + t.Logf("%+v", cr[0].Details) + assert.Equal(t, test.desc, cr[0].Details) + }) + } +} + +func TestDetector_UpdateSpam(t *testing.T) { + upd := &mocks.SampleUpdaterMock{ + AppendFunc: func(msg string) error { + return nil + }, + } + + d := NewDetector(Config{MaxAllowedEmoji: -1}) + d.WithSpamUpdater(upd) + + spamSamples := strings.NewReader("win free iPhone\nlottery prize xyz") + hamsSamples := strings.NewReader("hello world\nhow are you\nhave a good day") + lr, err := d.LoadSamples(strings.NewReader("xyz"), []io.Reader{spamSamples}, []io.Reader{hamsSamples}) + require.NoError(t, err) + assert.Equal(t, LoadResult{ExcludedTokens: 1, SpamSamples: 2, HamSamples: 3}, lr) + d.tokenizedSpam = nil // we don't need tokenizedSpam samples for this test + assert.Equal(t, 5, d.classifier.NAllDocument) + exp := map[string]map[Class]int{"win": {"spam": 1}, "free": {"spam": 1}, "iphone": {"spam": 1}, "lottery": {"spam": 1}, + "prize": {"spam": 1}, "hello": {"ham": 1}, "world": {"ham": 1}, "how": {"ham": 1}, "are": {"ham": 1}, "you": {"ham": 1}, + "have": {"ham": 1}, "good": {"ham": 1}, "day": {"ham": 1}} + assert.Equal(t, exp, d.classifier.LearningResults) + + msg := "another good world one iphone user writes good things day" + t.Run("initially a little bit ham", func(t *testing.T) { + spam, cr := d.Check(msg, 0) + assert.Equal(t, false, spam) + require.Len(t, cr, 1) + assert.Equal(t, "classifier", cr[0].Name) + assert.Equal(t, false, cr[0].Spam) + assert.Equal(t, "spam:-26.2365, ham:-25.8321", cr[0].Details) + }) + + err = d.UpdateSpam("another user writes") + assert.NoError(t, err) + assert.Equal(t, 6, d.classifier.NAllDocument) + assert.Equal(t, 1, len(upd.AppendCalls())) + + t.Run("after update mostly spam", func(t *testing.T) { + spam, cr := d.Check(msg, 0) + assert.Equal(t, true, spam) + require.Len(t, cr, 1) + assert.Equal(t, "classifier", cr[0].Name) + assert.Equal(t, true, cr[0].Spam) + assert.Equal(t, "spam:-26.5230, ham:-27.2162", cr[0].Details) + }) +} + +func TestDetector_UpdateHam(t *testing.T) { + upd := &mocks.SampleUpdaterMock{ + AppendFunc: func(msg string) error { + return nil + }, + } + + d := NewDetector(Config{MaxAllowedEmoji: -1}) + d.WithHamUpdater(upd) + + spamSamples := strings.NewReader("win free iPhone\nlottery prize xyz") + hamsSamples := strings.NewReader("hello world\nhow are you\nhave a good day") + lr, err := d.LoadSamples(strings.NewReader("xyz"), []io.Reader{spamSamples}, []io.Reader{hamsSamples}) + require.NoError(t, err) + assert.Equal(t, LoadResult{ExcludedTokens: 1, SpamSamples: 2, HamSamples: 3}, lr) + d.tokenizedSpam = nil // we don't need tokenizedSpam samples for this test + assert.Equal(t, 5, d.classifier.NAllDocument) + exp := map[string]map[Class]int{"win": {"spam": 1}, "free": {"spam": 1}, "iphone": {"spam": 1}, "lottery": {"spam": 1}, + "prize": {"spam": 1}, "hello": {"ham": 1}, "world": {"ham": 1}, "how": {"ham": 1}, "are": {"ham": 1}, "you": {"ham": 1}, + "have": {"ham": 1}, "good": {"ham": 1}, "day": {"ham": 1}} + assert.Equal(t, exp, d.classifier.LearningResults) + + msg := "another free good world one iphone user writes good things day" + t.Run("initially a little bit spam", func(t *testing.T) { + spam, cr := d.Check(msg, 0) + assert.Equal(t, true, spam) + require.Len(t, cr, 1) + assert.Equal(t, "classifier", cr[0].Name) + assert.Equal(t, true, cr[0].Spam) + assert.Equal(t, "spam:-28.4337, ham:-28.8766", cr[0].Details) + }) + + err = d.UpdateHam("another writes things") + assert.NoError(t, err) + assert.Equal(t, 6, d.classifier.NAllDocument) + assert.Equal(t, 1, len(upd.AppendCalls())) + + t.Run("after update mostly spam", func(t *testing.T) { + spam, cr := d.Check(msg, 0) + assert.Equal(t, false, spam) + require.Len(t, cr, 1) + assert.Equal(t, "classifier", cr[0].Name) + assert.Equal(t, false, cr[0].Spam) + assert.Equal(t, "spam:-30.1575, ham:-29.2050", cr[0].Details) + }) +} + +func TestDetector_Reset(t *testing.T) { + d := NewDetector(Config{}) + spamSamples := strings.NewReader("win free iPhone\nlottery prize xyz") + hamSamples := strings.NewReader("hello world\nhow are you\nhave a good day") + lr, err := d.LoadSamples(strings.NewReader("xyz"), []io.Reader{spamSamples}, []io.Reader{hamSamples}) + require.NoError(t, err) + assert.Equal(t, LoadResult{ExcludedTokens: 1, SpamSamples: 2, HamSamples: 3}, lr) + sr, err := d.LoadStopWords(strings.NewReader("в личку\nвсем привет")) + require.NoError(t, err) + assert.Equal(t, LoadResult{StopWords: 2}, sr) + + assert.Equal(t, 5, d.classifier.NAllDocument) + assert.Equal(t, 2, len(d.tokenizedSpam)) + assert.Equal(t, 1, len(d.excludedTokens)) + assert.Equal(t, 2, len(d.stopWords)) + + d.Reset() + assert.Equal(t, 0, d.classifier.NAllDocument) + assert.Equal(t, 0, len(d.tokenizedSpam)) + assert.Equal(t, 0, len(d.excludedTokens)) + assert.Equal(t, 0, len(d.stopWords)) +} diff --git a/lib/emoji.go b/lib/emoji.go new file mode 100644 index 00000000..c63c0ef1 --- /dev/null +++ b/lib/emoji.go @@ -0,0 +1,14 @@ +package lib + +import "regexp" + +func cleanEmoji(s string) string { + return emojiPattern.ReplaceAllString(s, "") +} + +func countEmoji(s string) int { + return len(emojiPattern.FindAllString(s, -1)) +} + +// borrowed from https://stackoverflow.com/a/72255061 +var emojiPattern = regexp.MustCompile(`[#*0-9]\x{FE0F}?\x{20E3}|©\x{FE0F}?|[®\x{203C}\x{2049}\x{2122}\x{2139}\x{2194}-\x{2199}\x{21A9}\x{21AA}]\x{FE0F}?|[\x{231A}\x{231B}]|[\x{2328}\x{23CF}]\x{FE0F}?|[\x{23E9}-\x{23EC}]|[\x{23ED}-\x{23EF}]\x{FE0F}?|\x{23F0}|[\x{23F1}\x{23F2}]\x{FE0F}?|\x{23F3}|[\x{23F8}-\x{23FA}\x{24C2}\x{25AA}\x{25AB}\x{25B6}\x{25C0}\x{25FB}\x{25FC}]\x{FE0F}?|[\x{25FD}\x{25FE}]|[\x{2600}-\x{2604}\x{260E}\x{2611}]\x{FE0F}?|[\x{2614}\x{2615}]|\x{2618}\x{FE0F}?|\x{261D}[\x{FE0F}\x{1F3FB}-\x{1F3FF}]?|[\x{2620}\x{2622}\x{2623}\x{2626}\x{262A}\x{262E}\x{262F}\x{2638}-\x{263A}\x{2640}\x{2642}]\x{FE0F}?|[\x{2648}-\x{2653}]|[\x{265F}\x{2660}\x{2663}\x{2665}\x{2666}\x{2668}\x{267B}\x{267E}]\x{FE0F}?|\x{267F}|\x{2692}\x{FE0F}?|\x{2693}|[\x{2694}-\x{2697}\x{2699}\x{269B}\x{269C}\x{26A0}]\x{FE0F}?|\x{26A1}|\x{26A7}\x{FE0F}?|[\x{26AA}\x{26AB}]|[\x{26B0}\x{26B1}]\x{FE0F}?|[\x{26BD}\x{26BE}\x{26C4}\x{26C5}]|\x{26C8}\x{FE0F}?|\x{26CE}|[\x{26CF}\x{26D1}\x{26D3}]\x{FE0F}?|\x{26D4}|\x{26E9}\x{FE0F}?|\x{26EA}|[\x{26F0}\x{26F1}]\x{FE0F}?|[\x{26F2}\x{26F3}]|\x{26F4}\x{FE0F}?|\x{26F5}|[\x{26F7}\x{26F8}]\x{FE0F}?|\x{26F9}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{FE0F}\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{26FA}\x{26FD}]|\x{2702}\x{FE0F}?|\x{2705}|[\x{2708}\x{2709}]\x{FE0F}?|[\x{270A}\x{270B}][\x{1F3FB}-\x{1F3FF}]?|[\x{270C}\x{270D}][\x{FE0F}\x{1F3FB}-\x{1F3FF}]?|\x{270F}\x{FE0F}?|[\x{2712}\x{2714}\x{2716}\x{271D}\x{2721}]\x{FE0F}?|\x{2728}|[\x{2733}\x{2734}\x{2744}\x{2747}]\x{FE0F}?|[\x{274C}\x{274E}\x{2753}-\x{2755}\x{2757}]|\x{2763}\x{FE0F}?|\x{2764}(?:\x{200D}[\x{1F525}\x{1FA79}]|\x{FE0F}(?:\x{200D}[\x{1F525}\x{1FA79}])?)?|[\x{2795}-\x{2797}]|\x{27A1}\x{FE0F}?|[\x{27B0}\x{27BF}]|[\x{2934}\x{2935}\x{2B05}-\x{2B07}]\x{FE0F}?|[\x{2B1B}\x{2B1C}\x{2B50}\x{2B55}]|[\x{3030}\x{303D}\x{3297}\x{3299}]\x{FE0F}?|[\x{1F004}\x{1F0CF}]|[\x{1F170}\x{1F171}\x{1F17E}\x{1F17F}]\x{FE0F}?|[\x{1F18E}\x{1F191}-\x{1F19A}]|\x{1F1E6}[\x{1F1E8}-\x{1F1EC}\x{1F1EE}\x{1F1F1}\x{1F1F2}\x{1F1F4}\x{1F1F6}-\x{1F1FA}\x{1F1FC}\x{1F1FD}\x{1F1FF}]|\x{1F1E7}[\x{1F1E6}\x{1F1E7}\x{1F1E9}-\x{1F1EF}\x{1F1F1}-\x{1F1F4}\x{1F1F6}-\x{1F1F9}\x{1F1FB}\x{1F1FC}\x{1F1FE}\x{1F1FF}]|\x{1F1E8}[\x{1F1E6}\x{1F1E8}\x{1F1E9}\x{1F1EB}-\x{1F1EE}\x{1F1F0}-\x{1F1F5}\x{1F1F7}\x{1F1FA}-\x{1F1FF}]|\x{1F1E9}[\x{1F1EA}\x{1F1EC}\x{1F1EF}\x{1F1F0}\x{1F1F2}\x{1F1F4}\x{1F1FF}]|\x{1F1EA}[\x{1F1E6}\x{1F1E8}\x{1F1EA}\x{1F1EC}\x{1F1ED}\x{1F1F7}-\x{1F1FA}]|\x{1F1EB}[\x{1F1EE}-\x{1F1F0}\x{1F1F2}\x{1F1F4}\x{1F1F7}]|\x{1F1EC}[\x{1F1E6}\x{1F1E7}\x{1F1E9}-\x{1F1EE}\x{1F1F1}-\x{1F1F3}\x{1F1F5}-\x{1F1FA}\x{1F1FC}\x{1F1FE}]|\x{1F1ED}[\x{1F1F0}\x{1F1F2}\x{1F1F3}\x{1F1F7}\x{1F1F9}\x{1F1FA}]|\x{1F1EE}[\x{1F1E8}-\x{1F1EA}\x{1F1F1}-\x{1F1F4}\x{1F1F6}-\x{1F1F9}]|\x{1F1EF}[\x{1F1EA}\x{1F1F2}\x{1F1F4}\x{1F1F5}]|\x{1F1F0}[\x{1F1EA}\x{1F1EC}-\x{1F1EE}\x{1F1F2}\x{1F1F3}\x{1F1F5}\x{1F1F7}\x{1F1FC}\x{1F1FE}\x{1F1FF}]|\x{1F1F1}[\x{1F1E6}-\x{1F1E8}\x{1F1EE}\x{1F1F0}\x{1F1F7}-\x{1F1FB}\x{1F1FE}]|\x{1F1F2}[\x{1F1E6}\x{1F1E8}-\x{1F1ED}\x{1F1F0}-\x{1F1FF}]|\x{1F1F3}[\x{1F1E6}\x{1F1E8}\x{1F1EA}-\x{1F1EC}\x{1F1EE}\x{1F1F1}\x{1F1F4}\x{1F1F5}\x{1F1F7}\x{1F1FA}\x{1F1FF}]|\x{1F1F4}\x{1F1F2}|\x{1F1F5}[\x{1F1E6}\x{1F1EA}-\x{1F1ED}\x{1F1F0}-\x{1F1F3}\x{1F1F7}-\x{1F1F9}\x{1F1FC}\x{1F1FE}]|\x{1F1F6}\x{1F1E6}|\x{1F1F7}[\x{1F1EA}\x{1F1F4}\x{1F1F8}\x{1F1FA}\x{1F1FC}]|\x{1F1F8}[\x{1F1E6}-\x{1F1EA}\x{1F1EC}-\x{1F1F4}\x{1F1F7}-\x{1F1F9}\x{1F1FB}\x{1F1FD}-\x{1F1FF}]|\x{1F1F9}[\x{1F1E6}\x{1F1E8}\x{1F1E9}\x{1F1EB}-\x{1F1ED}\x{1F1EF}-\x{1F1F4}\x{1F1F7}\x{1F1F9}\x{1F1FB}\x{1F1FC}\x{1F1FF}]|\x{1F1FA}[\x{1F1E6}\x{1F1EC}\x{1F1F2}\x{1F1F3}\x{1F1F8}\x{1F1FE}\x{1F1FF}]|\x{1F1FB}[\x{1F1E6}\x{1F1E8}\x{1F1EA}\x{1F1EC}\x{1F1EE}\x{1F1F3}\x{1F1FA}]|\x{1F1FC}[\x{1F1EB}\x{1F1F8}]|\x{1F1FD}\x{1F1F0}|\x{1F1FE}[\x{1F1EA}\x{1F1F9}]|\x{1F1FF}[\x{1F1E6}\x{1F1F2}\x{1F1FC}]|\x{1F201}|\x{1F202}\x{FE0F}?|[\x{1F21A}\x{1F22F}\x{1F232}-\x{1F236}]|\x{1F237}\x{FE0F}?|[\x{1F238}-\x{1F23A}\x{1F250}\x{1F251}\x{1F300}-\x{1F320}]|[\x{1F321}\x{1F324}-\x{1F32C}]\x{FE0F}?|[\x{1F32D}-\x{1F335}]|\x{1F336}\x{FE0F}?|[\x{1F337}-\x{1F37C}]|\x{1F37D}\x{FE0F}?|[\x{1F37E}-\x{1F384}]|\x{1F385}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F386}-\x{1F393}]|[\x{1F396}\x{1F397}\x{1F399}-\x{1F39B}\x{1F39E}\x{1F39F}]\x{FE0F}?|[\x{1F3A0}-\x{1F3C1}]|\x{1F3C2}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F3C3}\x{1F3C4}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F3C5}\x{1F3C6}]|\x{1F3C7}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F3C8}\x{1F3C9}]|\x{1F3CA}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F3CB}\x{1F3CC}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{FE0F}\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F3CD}\x{1F3CE}]\x{FE0F}?|[\x{1F3CF}-\x{1F3D3}]|[\x{1F3D4}-\x{1F3DF}]\x{FE0F}?|[\x{1F3E0}-\x{1F3F0}]|\x{1F3F3}(?:\x{200D}(?:\x{26A7}\x{FE0F}?|\x{1F308})|\x{FE0F}(?:\x{200D}(?:\x{26A7}\x{FE0F}?|\x{1F308}))?)?|\x{1F3F4}(?:\x{200D}\x{2620}\x{FE0F}?|\x{E0067}\x{E0062}(?:\x{E0065}\x{E006E}\x{E0067}|\x{E0073}\x{E0063}\x{E0074}|\x{E0077}\x{E006C}\x{E0073})\x{E007F})?|[\x{1F3F5}\x{1F3F7}]\x{FE0F}?|[\x{1F3F8}-\x{1F407}]|\x{1F408}(?:\x{200D}\x{2B1B})?|[\x{1F409}-\x{1F414}]|\x{1F415}(?:\x{200D}\x{1F9BA})?|[\x{1F416}-\x{1F43A}]|\x{1F43B}(?:\x{200D}\x{2744}\x{FE0F}?)?|[\x{1F43C}-\x{1F43E}]|\x{1F43F}\x{FE0F}?|\x{1F440}|\x{1F441}(?:\x{200D}\x{1F5E8}\x{FE0F}?|\x{FE0F}(?:\x{200D}\x{1F5E8}\x{FE0F}?)?)?|[\x{1F442}\x{1F443}][\x{1F3FB}-\x{1F3FF}]?|[\x{1F444}\x{1F445}]|[\x{1F446}-\x{1F450}][\x{1F3FB}-\x{1F3FF}]?|[\x{1F451}-\x{1F465}]|[\x{1F466}\x{1F467}][\x{1F3FB}-\x{1F3FF}]?|\x{1F468}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D})?\x{1F468}|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}]|\x{1F466}(?:\x{200D}\x{1F466})?|\x{1F467}(?:\x{200D}[\x{1F466}\x{1F467}])?|[\x{1F468}\x{1F469}]\x{200D}(?:\x{1F466}(?:\x{200D}\x{1F466})?|\x{1F467}(?:\x{200D}[\x{1F466}\x{1F467}])?)|[\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}])|\x{1F3FB}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D})?\x{1F468}[\x{1F3FB}-\x{1F3FF}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F468}[\x{1F3FC}-\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FC}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D})?\x{1F468}[\x{1F3FB}-\x{1F3FF}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F468}[\x{1F3FB}\x{1F3FD}-\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FD}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D})?\x{1F468}[\x{1F3FB}-\x{1F3FF}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F468}[\x{1F3FB}\x{1F3FC}\x{1F3FE}\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FE}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D})?\x{1F468}[\x{1F3FB}-\x{1F3FF}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F468}[\x{1F3FB}-\x{1F3FD}\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FF}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D})?\x{1F468}[\x{1F3FB}-\x{1F3FF}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F468}[\x{1F3FB}-\x{1F3FE}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?)?|\x{1F469}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D})?[\x{1F468}\x{1F469}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}]|\x{1F466}(?:\x{200D}\x{1F466})?|\x{1F467}(?:\x{200D}[\x{1F466}\x{1F467}])?|\x{1F469}\x{200D}(?:\x{1F466}(?:\x{200D}\x{1F466})?|\x{1F467}(?:\x{200D}[\x{1F466}\x{1F467}])?)|[\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}])|\x{1F3FB}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FF}]|\x{1F48B}\x{200D}[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FF}])|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}[\x{1F468}\x{1F469}][\x{1F3FC}-\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FC}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FF}]|\x{1F48B}\x{200D}[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FF}])|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}[\x{1F468}\x{1F469}][\x{1F3FB}\x{1F3FD}-\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FD}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FF}]|\x{1F48B}\x{200D}[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FF}])|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}[\x{1F468}\x{1F469}][\x{1F3FB}\x{1F3FC}\x{1F3FE}\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FE}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FF}]|\x{1F48B}\x{200D}[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FF}])|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FD}\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FF}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FF}]|\x{1F48B}\x{200D}[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FF}])|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}[\x{1F468}\x{1F469}][\x{1F3FB}-\x{1F3FE}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?)?|\x{1F46A}|[\x{1F46B}-\x{1F46D}][\x{1F3FB}-\x{1F3FF}]?|\x{1F46E}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F46F}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?|[\x{1F470}\x{1F471}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F472}[\x{1F3FB}-\x{1F3FF}]?|\x{1F473}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F474}-\x{1F476}][\x{1F3FB}-\x{1F3FF}]?|\x{1F477}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F478}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F479}-\x{1F47B}]|\x{1F47C}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F47D}-\x{1F480}]|[\x{1F481}\x{1F482}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F483}[\x{1F3FB}-\x{1F3FF}]?|\x{1F484}|\x{1F485}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F486}\x{1F487}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F488}-\x{1F48E}]|\x{1F48F}[\x{1F3FB}-\x{1F3FF}]?|\x{1F490}|\x{1F491}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F492}-\x{1F4A9}]|\x{1F4AA}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F4AB}-\x{1F4FC}]|\x{1F4FD}\x{FE0F}?|[\x{1F4FF}-\x{1F53D}]|[\x{1F549}\x{1F54A}]\x{FE0F}?|[\x{1F54B}-\x{1F54E}\x{1F550}-\x{1F567}]|[\x{1F56F}\x{1F570}\x{1F573}]\x{FE0F}?|\x{1F574}[\x{FE0F}\x{1F3FB}-\x{1F3FF}]?|\x{1F575}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{FE0F}\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F576}-\x{1F579}]\x{FE0F}?|\x{1F57A}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F587}\x{1F58A}-\x{1F58D}]\x{FE0F}?|\x{1F590}[\x{FE0F}\x{1F3FB}-\x{1F3FF}]?|[\x{1F595}\x{1F596}][\x{1F3FB}-\x{1F3FF}]?|\x{1F5A4}|[\x{1F5A5}\x{1F5A8}\x{1F5B1}\x{1F5B2}\x{1F5BC}\x{1F5C2}-\x{1F5C4}\x{1F5D1}-\x{1F5D3}\x{1F5DC}-\x{1F5DE}\x{1F5E1}\x{1F5E3}\x{1F5E8}\x{1F5EF}\x{1F5F3}\x{1F5FA}]\x{FE0F}?|[\x{1F5FB}-\x{1F62D}]|\x{1F62E}(?:\x{200D}\x{1F4A8})?|[\x{1F62F}-\x{1F634}]|\x{1F635}(?:\x{200D}\x{1F4AB})?|\x{1F636}(?:\x{200D}\x{1F32B}\x{FE0F}?)?|[\x{1F637}-\x{1F644}]|[\x{1F645}-\x{1F647}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F648}-\x{1F64A}]|\x{1F64B}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F64C}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F64D}\x{1F64E}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F64F}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F680}-\x{1F6A2}]|\x{1F6A3}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F6A4}-\x{1F6B3}]|[\x{1F6B4}-\x{1F6B6}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F6B7}-\x{1F6BF}]|\x{1F6C0}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F6C1}-\x{1F6C5}]|\x{1F6CB}\x{FE0F}?|\x{1F6CC}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F6CD}-\x{1F6CF}]\x{FE0F}?|[\x{1F6D0}-\x{1F6D2}\x{1F6D5}-\x{1F6D7}\x{1F6DD}-\x{1F6DF}]|[\x{1F6E0}-\x{1F6E5}\x{1F6E9}]\x{FE0F}?|[\x{1F6EB}\x{1F6EC}]|[\x{1F6F0}\x{1F6F3}]\x{FE0F}?|[\x{1F6F4}-\x{1F6FC}\x{1F7E0}-\x{1F7EB}\x{1F7F0}]|\x{1F90C}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F90D}\x{1F90E}]|\x{1F90F}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F910}-\x{1F917}]|[\x{1F918}-\x{1F91F}][\x{1F3FB}-\x{1F3FF}]?|[\x{1F920}-\x{1F925}]|\x{1F926}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F927}-\x{1F92F}]|[\x{1F930}-\x{1F934}][\x{1F3FB}-\x{1F3FF}]?|\x{1F935}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F936}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F937}-\x{1F939}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F93A}|\x{1F93C}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?|[\x{1F93D}\x{1F93E}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F93F}-\x{1F945}\x{1F947}-\x{1F976}]|\x{1F977}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F978}-\x{1F9B4}]|[\x{1F9B5}\x{1F9B6}][\x{1F3FB}-\x{1F3FF}]?|\x{1F9B7}|[\x{1F9B8}\x{1F9B9}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F9BA}|\x{1F9BB}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F9BC}-\x{1F9CC}]|[\x{1F9CD}-\x{1F9CF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F9D0}|\x{1F9D1}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F384}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F9D1}|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}])|\x{1F3FB}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D}|)\x{1F9D1}[\x{1F3FC}-\x{1F3FF}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F384}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F9D1}[\x{1F3FB}-\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FC}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D}|)\x{1F9D1}[\x{1F3FB}\x{1F3FD}-\x{1F3FF}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F384}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F9D1}[\x{1F3FB}-\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FD}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D}|)\x{1F9D1}[\x{1F3FB}\x{1F3FC}\x{1F3FE}\x{1F3FF}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F384}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F9D1}[\x{1F3FB}-\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FE}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D}|)\x{1F9D1}[\x{1F3FB}-\x{1F3FD}\x{1F3FF}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F384}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F9D1}[\x{1F3FB}-\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?|\x{1F3FF}(?:\x{200D}(?:[\x{2695}\x{2696}\x{2708}]\x{FE0F}?|\x{2764}\x{FE0F}?\x{200D}(?:\x{1F48B}\x{200D}|)\x{1F9D1}[\x{1F3FB}-\x{1F3FE}]|[\x{1F33E}\x{1F373}\x{1F37C}\x{1F384}\x{1F393}\x{1F3A4}\x{1F3A8}\x{1F3EB}\x{1F3ED}\x{1F4BB}\x{1F4BC}\x{1F527}\x{1F52C}\x{1F680}\x{1F692}]|\x{1F91D}\x{200D}\x{1F9D1}[\x{1F3FB}-\x{1F3FF}]|[\x{1F9AF}-\x{1F9B3}\x{1F9BC}\x{1F9BD}]))?)?|[\x{1F9D2}\x{1F9D3}][\x{1F3FB}-\x{1F3FF}]?|\x{1F9D4}(?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|\x{1F9D5}[\x{1F3FB}-\x{1F3FF}]?|[\x{1F9D6}-\x{1F9DD}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?|[\x{1F3FB}-\x{1F3FF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?)?|[\x{1F9DE}\x{1F9DF}](?:\x{200D}[\x{2640}\x{2642}]\x{FE0F}?)?|[\x{1F9E0}-\x{1F9FF}\x{1FA70}-\x{1FA74}\x{1FA78}-\x{1FA7C}\x{1FA80}-\x{1FA86}\x{1FA90}-\x{1FAAC}\x{1FAB0}-\x{1FABA}\x{1FAC0}-\x{1FAC2}]|[\x{1FAC3}-\x{1FAC5}][\x{1F3FB}-\x{1F3FF}]?|[\x{1FAD0}-\x{1FAD9}\x{1FAE0}-\x{1FAE7}]|\x{1FAF0}[\x{1F3FB}-\x{1F3FF}]?|\x{1FAF1}(?:\x{1F3FB}(?:\x{200D}\x{1FAF2}[\x{1F3FC}-\x{1F3FF}])?|\x{1F3FC}(?:\x{200D}\x{1FAF2}[\x{1F3FB}\x{1F3FD}-\x{1F3FF}])?|\x{1F3FD}(?:\x{200D}\x{1FAF2}[\x{1F3FB}\x{1F3FC}\x{1F3FE}\x{1F3FF}])?|\x{1F3FE}(?:\x{200D}\x{1FAF2}[\x{1F3FB}-\x{1F3FD}\x{1F3FF}])?|\x{1F3FF}(?:\x{200D}\x{1FAF2}[\x{1F3FB}-\x{1F3FE}])?)?|[\x{1FAF2}-\x{1FAF6}][\x{1F3FB}-\x{1F3FF}]?`) diff --git a/lib/emoji_test.go b/lib/emoji_test.go new file mode 100644 index 00000000..5776fefe --- /dev/null +++ b/lib/emoji_test.go @@ -0,0 +1,55 @@ +package lib + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +//nolint:stylecheck // it has unicode symbols purposely +func Test_countEmoji(t *testing.T) { + tests := []struct { + name string + input string + count int + }{ + {"NoEmoji", "Hello, world!", 0}, + {"OneEmoji", "Hi there 👋", 1}, + {"TwoEmojis", "Good morning 🌞🌻", 2}, + {"Mixed", "👨‍👩‍👧‍👦 Family emoji", 1}, + {"EmojiSequences", "🏳️‍🌈 Rainbow flag", 1}, + {"TextAfterEmoji", "😊 Have a nice day!", 1}, + {"OnlyEmojis", "😁🐶🍕", 3}, + {"WithCyrillic", "Привет 🌞 🍕 мир! 👋", 3}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.count, countEmoji(tt.input)) + }) + } +} + +//nolint:stylecheck // it has unicode symbols purposely +func Test_cleanEmoji(t *testing.T) { + tests := []struct { + name string + input string + clean string + }{ + {"NoEmoji", "Hello, world!", "Hello, world!"}, + {"OneEmoji", "Hi there 👋", "Hi there "}, + {"TwoEmojis", "Good morning 🌞🌻", "Good morning "}, + {"Mixed", "👨‍👩‍👧‍👦 Family emoji", " Family emoji"}, + {"EmojiSequences", "🏳️‍🌈 Rainbow flag", " Rainbow flag"}, + {"TextAfterEmoji", "😊 Have a nice day!", " Have a nice day!"}, + {"OnlyEmojis", "😁🐶🍕", ""}, + {"WithCyrillic", "Привет 🌞 🍕 мир! 👋", "Привет мир! "}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.clean, cleanEmoji(tt.input)) + }) + } +} diff --git a/lib/lib.go b/lib/lib.go new file mode 100644 index 00000000..1c1f9a2f --- /dev/null +++ b/lib/lib.go @@ -0,0 +1,44 @@ +// Package lib provides functionality for spam detection. The primary type in this package +// is the Detector, which is used to identify spam in given texts. It is initialized with +// parameters defined in the Config struct. +// +// The Detector is designed to be thread-safe and supports concurrent usage. +// +// Before using a Detector, it is necessary to load spam data using one of the Load* methods: +// +// - LoadStopWords: This method loads stop-words (stop-phrases) from provided readers. The reader can +// parse words either as one word (or phrase) per line or as a comma-separated list of words +// (phrases) enclosed in double quotes. Both formats can be mixed within the same reader. +// Example of a reader stream: +// "word1" +// "word2" +// "hello world" +// "some phrase", "another phrase" +// +// - LoadSamples: This method loads samples of spam and ham (non-spam) messages. It also +// accepts a reader for a list of excluded tokens, often comprising words too common to aid +// in spam detection. The loaded samples are utilized to train the spam detectors, which include +// one based on the Naive Bayes algorithm and another on Cosine Similarity. +// +// Additionally, Config provides configuration options: +// +// - Config.MaxAllowedEmoji specifies the maximum number of emojis permissible in a message. +// Messages exceeding this count are marked as spam. A negative value deactivates emoji detection. +// +// - Config.MinMsgLen defines the minimum message length for spam checks. Messages shorter +// than this threshold are ignored. A negative value or zero deactivates this check. +// +// - Config.FirstMessageOnly specifies whether only the first message from a given userID should +// be checked. +// +// - Config.CasAPI specifies the URL of the CAS API to use for spam detection. If this is empty, the +// detector will not use the CAS API checks. +// +// - Config.HTTPClient specifies the HTTP client to use for CAS API checks. This interface is satisfied +// by the standard library's http.Client type. +// +// Other important methods are Detector.UpdateSpam and Detector.UpdateHam, which are used to update the +// spam and ham samples on the fly. Those methods are thread-safe and can be called concurrently. +// To call them Detector.WithSpamUpdater and Detector.WithHamUpdater methods should be used first to provide +// user-defined structs that implement the SampleUpdater interface. +package lib diff --git a/app/bot/mocks/http_client.go b/lib/mocks/http_client.go similarity index 71% rename from app/bot/mocks/http_client.go rename to lib/mocks/http_client.go index 6bbefa58..38ffe634 100644 --- a/app/bot/mocks/http_client.go +++ b/lib/mocks/http_client.go @@ -8,22 +8,22 @@ import ( "sync" ) -// HTTPClient is a mock implementation of bot.HTTPClient. +// HTTPClientMock is a mock implementation of lib.HTTPClient. // // func TestSomethingThatUsesHTTPClient(t *testing.T) { // -// // make and configure a mocked bot.HTTPClient -// mockedHTTPClient := &HTTPClient{ +// // make and configure a mocked lib.HTTPClient +// mockedHTTPClient := &HTTPClientMock{ // DoFunc: func(req *http.Request) (*http.Response, error) { // panic("mock out the Do method") // }, // } // -// // use mockedHTTPClient in code that requires bot.HTTPClient +// // use mockedHTTPClient in code that requires lib.HTTPClient // // and then make assertions. // // } -type HTTPClient struct { +type HTTPClientMock struct { // DoFunc mocks the Do method. DoFunc func(req *http.Request) (*http.Response, error) @@ -39,9 +39,9 @@ type HTTPClient struct { } // Do calls DoFunc. -func (mock *HTTPClient) Do(req *http.Request) (*http.Response, error) { +func (mock *HTTPClientMock) Do(req *http.Request) (*http.Response, error) { if mock.DoFunc == nil { - panic("HTTPClient.DoFunc: method is nil but HTTPClient.Do was just called") + panic("HTTPClientMock.DoFunc: method is nil but HTTPClient.Do was just called") } callInfo := struct { Req *http.Request @@ -58,7 +58,7 @@ func (mock *HTTPClient) Do(req *http.Request) (*http.Response, error) { // Check the length with: // // len(mockedHTTPClient.DoCalls()) -func (mock *HTTPClient) DoCalls() []struct { +func (mock *HTTPClientMock) DoCalls() []struct { Req *http.Request } { var calls []struct { diff --git a/lib/mocks/sample_updater.go b/lib/mocks/sample_updater.go new file mode 100644 index 00000000..2f551f47 --- /dev/null +++ b/lib/mocks/sample_updater.go @@ -0,0 +1,108 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package mocks + +import ( + "io" + "sync" +) + +// SampleUpdaterMock is a mock implementation of lib.SampleUpdater. +// +// func TestSomethingThatUsesSampleUpdater(t *testing.T) { +// +// // make and configure a mocked lib.SampleUpdater +// mockedSampleUpdater := &SampleUpdaterMock{ +// AppendFunc: func(msg string) error { +// panic("mock out the Append method") +// }, +// ReaderFunc: func() (io.ReadCloser, error) { +// panic("mock out the Reader method") +// }, +// } +// +// // use mockedSampleUpdater in code that requires lib.SampleUpdater +// // and then make assertions. +// +// } +type SampleUpdaterMock struct { + // AppendFunc mocks the Append method. + AppendFunc func(msg string) error + + // ReaderFunc mocks the Reader method. + ReaderFunc func() (io.ReadCloser, error) + + // calls tracks calls to the methods. + calls struct { + // Append holds details about calls to the Append method. + Append []struct { + // Msg is the msg argument value. + Msg string + } + // Reader holds details about calls to the Reader method. + Reader []struct { + } + } + lockAppend sync.RWMutex + lockReader sync.RWMutex +} + +// Append calls AppendFunc. +func (mock *SampleUpdaterMock) Append(msg string) error { + if mock.AppendFunc == nil { + panic("SampleUpdaterMock.AppendFunc: method is nil but SampleUpdater.Append was just called") + } + callInfo := struct { + Msg string + }{ + Msg: msg, + } + mock.lockAppend.Lock() + mock.calls.Append = append(mock.calls.Append, callInfo) + mock.lockAppend.Unlock() + return mock.AppendFunc(msg) +} + +// AppendCalls gets all the calls that were made to Append. +// Check the length with: +// +// len(mockedSampleUpdater.AppendCalls()) +func (mock *SampleUpdaterMock) AppendCalls() []struct { + Msg string +} { + var calls []struct { + Msg string + } + mock.lockAppend.RLock() + calls = mock.calls.Append + mock.lockAppend.RUnlock() + return calls +} + +// Reader calls ReaderFunc. +func (mock *SampleUpdaterMock) Reader() (io.ReadCloser, error) { + if mock.ReaderFunc == nil { + panic("SampleUpdaterMock.ReaderFunc: method is nil but SampleUpdater.Reader was just called") + } + callInfo := struct { + }{} + mock.lockReader.Lock() + mock.calls.Reader = append(mock.calls.Reader, callInfo) + mock.lockReader.Unlock() + return mock.ReaderFunc() +} + +// ReaderCalls gets all the calls that were made to Reader. +// Check the length with: +// +// len(mockedSampleUpdater.ReaderCalls()) +func (mock *SampleUpdaterMock) ReaderCalls() []struct { +} { + var calls []struct { + } + mock.lockReader.RLock() + calls = mock.calls.Reader + mock.lockReader.RUnlock() + return calls +} diff --git a/site/docs/index.md b/site/docs/index.md index 0b906d89..70a8a4c5 100644 --- a/site/docs/index.md +++ b/site/docs/index.md @@ -84,6 +84,12 @@ To allow such a feature, some parameters in `admin` section must be specified: - `--admin.group=, [$ADMIN_GROUP]` - admin chat/group name/id. This can be a group name (for public groups), but usually it is a group id (for private groups) or personal accounts. - `--admin.secret=, [$ADMIN_SECRET]` - admin secret. This is a secret string to protect generated links. It is recommended to set it to some random, long string. +### Updating spam samples dynamically + +The bot can be configured to update spam samples dynamically. To enable this feature, reporting to the admin chat must be enabled (see `--admin.url=, [$ADMIN_URL]` above. If any of privileged users (`--super=, [$SUPER_USER]`) forwards a message to admin chat, the bot will add this message to the internal spam samples file (`spam-dynamic.txt`) and reload it. This allows the bot to learn new spam patterns on the fly. In addition, the bot will do the best to remove the original spam message from the group and ban the user who sent it. This is not always possible, as the forwarding strips the original user id. To address this limitation, tg-spam keeps the list of latest messages (in fact, it stores hashes) associated with the user id and the message id. This information is used to find the original message and ban the user. + +Note: if the bot is running in docker container, `--files.dynamic-spam=, [$FILES_DYNAMIC_SPAM]` must be set to the mapped volume's location to stay persistent after container restart. + ### Logging The default logging prints spam reports to the console (stdout). The bot can log all the spam messages to the file as well. To enable this feature, set `--logger.enabled, [$LOGGER_ENABLED]` to `true`. By default, the bot will log to the file `tg-spam.log` in the current directory. To change the location, set `--logger.file, [$LOGGER_FILE]` to the desired location. The bot will rotate the log file when it reaches the size specified in `--logger.max-size, [$LOGGER_MAX_SIZE]` (default is 100M). The bot will keep up to `--logger.max-backups, [$LOGGER_MAX_BACKUPS]` (default is 10) of the old, compressed log files. @@ -122,6 +128,7 @@ Use this token to access the HTTP API: ``` --testing-id= testing ids, allow bot to reply to them [$TESTING_ID] + --history-duration= history duration (default: 1h) [$HISTORY_DURATION] --super= super-users [$SUPER_USER] --no-spam-reply do not reply to spam messages [$NO_SPAM_REPLY] --similarity-threshold= spam threshold (default: 0.5) [$SIMILARITY_THRESHOLD] @@ -159,6 +166,8 @@ files: --files.samples-ham= path to ham samples (default: data/ham-samples.txt) [$FILES_SAMPLES_HAM] --files.exclude-tokens= path to exclude tokens file (default: data/exclude-tokens.txt) [$FILES_EXCLUDE_TOKENS] --files.stop-words= path to stop words file (default: data/stop-words.txt) [$FILES_STOP_WORDS] + --files.dynamic-spam= path to dynamic spam file (default: data/spam-dynamic.txt) [$FILES_DYNAMIC_SPAM] + --files.dynamic-ham= path to dynamic ham file (default: data/ham-dynamic.txt) [$FILES_DYNAMIC_HAM] message: --message.startup= startup message [$MESSAGE_STARTUP] @@ -168,4 +177,42 @@ message: Help Options: -h, --help Show this help message -``` \ No newline at end of file +``` + +## Using tg-spam as a library + +The bot can be used as a library as well. To do so, import the `github.com/umputun/tg-spam/lib` package and create a new instance of the `Detector` struct. Then, call the `Check` method with the message and userID to check. The method will return `true` if the message is spam and `false` otherwise. In addition, the `Check` method will return the list of applied rules as well as the spam-related details. + +For more details see the [TBD]() + +Example: + +```go +package main + +import ( + "io" + + tgspam "github.com/umputun/tg-spam/lib" +) + +func main() { + detector := tgspam.NewDetector(tgspam.Config{ + SimilarityThreshold: 0.5, + MinMsgLen: 50, + MaxEmoji: 2, + FirstMessageOnly: false, + HTTPClient: &http.Client{Timeout: 30 * time.Second}, + }) + + // prepare samples and exclude tokens + spamSample := bytes.NewBufferString("this is spam\nwin a prize\n") // need io.Reader, in real life it will be a file + hamSample := bytes.NewBufferString("this is ham\n") + excludeTokens := bytes.NewBufferString(`"a", "the"`) + + // load samples + detector.LoadSamples(excludeTokens, []io.Reader{spamSample}, []io.Reader{hamSample}) + + isSpam, details := detector.Check("this is spam", 123456) +} +``` diff --git a/vendor/github.com/aymerick/douceur/css/declaration.go b/vendor/github.com/aymerick/douceur/css/declaration.go deleted file mode 100644 index 61d29d33..00000000 --- a/vendor/github.com/aymerick/douceur/css/declaration.go +++ /dev/null @@ -1,60 +0,0 @@ -package css - -import "fmt" - -// Declaration represents a parsed style property -type Declaration struct { - Property string - Value string - Important bool -} - -// NewDeclaration instanciates a new Declaration -func NewDeclaration() *Declaration { - return &Declaration{} -} - -// Returns string representation of the Declaration -func (decl *Declaration) String() string { - return decl.StringWithImportant(true) -} - -// StringWithImportant returns string representation with optional !important part -func (decl *Declaration) StringWithImportant(option bool) string { - result := fmt.Sprintf("%s: %s", decl.Property, decl.Value) - - if option && decl.Important { - result += " !important" - } - - result += ";" - - return result -} - -// Equal returns true if both Declarations are equals -func (decl *Declaration) Equal(other *Declaration) bool { - return (decl.Property == other.Property) && (decl.Value == other.Value) && (decl.Important == other.Important) -} - -// -// DeclarationsByProperty -// - -// DeclarationsByProperty represents sortable style declarations -type DeclarationsByProperty []*Declaration - -// Implements sort.Interface -func (declarations DeclarationsByProperty) Len() int { - return len(declarations) -} - -// Implements sort.Interface -func (declarations DeclarationsByProperty) Swap(i, j int) { - declarations[i], declarations[j] = declarations[j], declarations[i] -} - -// Implements sort.Interface -func (declarations DeclarationsByProperty) Less(i, j int) bool { - return declarations[i].Property < declarations[j].Property -} diff --git a/vendor/github.com/aymerick/douceur/css/rule.go b/vendor/github.com/aymerick/douceur/css/rule.go deleted file mode 100644 index b5a44b54..00000000 --- a/vendor/github.com/aymerick/douceur/css/rule.go +++ /dev/null @@ -1,230 +0,0 @@ -package css - -import ( - "fmt" - "strings" -) - -const ( - indentSpace = 2 -) - -// RuleKind represents a Rule kind -type RuleKind int - -// Rule kinds -const ( - QualifiedRule RuleKind = iota - AtRule -) - -// At Rules than have Rules inside their block instead of Declarations -var atRulesWithRulesBlock = []string{ - "@document", "@font-feature-values", "@keyframes", "@media", "@supports", -} - -// Rule represents a parsed CSS rule -type Rule struct { - Kind RuleKind - - // At Rule name (eg: "@media") - Name string - - // Raw prelude - Prelude string - - // Qualified Rule selectors parsed from prelude - Selectors []string - - // Style properties - Declarations []*Declaration - - // At Rule embedded rules - Rules []*Rule - - // Current rule embedding level - EmbedLevel int -} - -// NewRule instanciates a new Rule -func NewRule(kind RuleKind) *Rule { - return &Rule{ - Kind: kind, - } -} - -// Returns string representation of rule kind -func (kind RuleKind) String() string { - switch kind { - case QualifiedRule: - return "Qualified Rule" - case AtRule: - return "At Rule" - default: - return "WAT" - } -} - -// EmbedsRules returns true if this rule embeds another rules -func (rule *Rule) EmbedsRules() bool { - if rule.Kind == AtRule { - for _, atRuleName := range atRulesWithRulesBlock { - if rule.Name == atRuleName { - return true - } - } - } - - return false -} - -// Equal returns true if both rules are equals -func (rule *Rule) Equal(other *Rule) bool { - if (rule.Kind != other.Kind) || - (rule.Prelude != other.Prelude) || - (rule.Name != other.Name) { - return false - } - - if (len(rule.Selectors) != len(other.Selectors)) || - (len(rule.Declarations) != len(other.Declarations)) || - (len(rule.Rules) != len(other.Rules)) { - return false - } - - for i, sel := range rule.Selectors { - if sel != other.Selectors[i] { - return false - } - } - - for i, decl := range rule.Declarations { - if !decl.Equal(other.Declarations[i]) { - return false - } - } - - for i, rule := range rule.Rules { - if !rule.Equal(other.Rules[i]) { - return false - } - } - - return true -} - -// Diff returns a string representation of rules differences -func (rule *Rule) Diff(other *Rule) []string { - result := []string{} - - if rule.Kind != other.Kind { - result = append(result, fmt.Sprintf("Kind: %s | %s", rule.Kind.String(), other.Kind.String())) - } - - if rule.Prelude != other.Prelude { - result = append(result, fmt.Sprintf("Prelude: \"%s\" | \"%s\"", rule.Prelude, other.Prelude)) - } - - if rule.Name != other.Name { - result = append(result, fmt.Sprintf("Name: \"%s\" | \"%s\"", rule.Name, other.Name)) - } - - if len(rule.Selectors) != len(other.Selectors) { - result = append(result, fmt.Sprintf("Selectors: %v | %v", strings.Join(rule.Selectors, ", "), strings.Join(other.Selectors, ", "))) - } else { - for i, sel := range rule.Selectors { - if sel != other.Selectors[i] { - result = append(result, fmt.Sprintf("Selector: \"%s\" | \"%s\"", sel, other.Selectors[i])) - } - } - } - - if len(rule.Declarations) != len(other.Declarations) { - result = append(result, fmt.Sprintf("Declarations Nb: %d | %d", len(rule.Declarations), len(other.Declarations))) - } else { - for i, decl := range rule.Declarations { - if !decl.Equal(other.Declarations[i]) { - result = append(result, fmt.Sprintf("Declaration: \"%s\" | \"%s\"", decl.String(), other.Declarations[i].String())) - } - } - } - - if len(rule.Rules) != len(other.Rules) { - result = append(result, fmt.Sprintf("Rules Nb: %d | %d", len(rule.Rules), len(other.Rules))) - } else { - - for i, rule := range rule.Rules { - if !rule.Equal(other.Rules[i]) { - result = append(result, fmt.Sprintf("Rule: \"%s\" | \"%s\"", rule.String(), other.Rules[i].String())) - } - } - } - - return result -} - -// Returns the string representation of a rule -func (rule *Rule) String() string { - result := "" - - if rule.Kind == QualifiedRule { - for i, sel := range rule.Selectors { - if i != 0 { - result += ", " - } - result += sel - } - } else { - // AtRule - result += fmt.Sprintf("%s", rule.Name) - - if rule.Prelude != "" { - if result != "" { - result += " " - } - result += fmt.Sprintf("%s", rule.Prelude) - } - } - - if (len(rule.Declarations) == 0) && (len(rule.Rules) == 0) { - result += ";" - } else { - result += " {\n" - - if rule.EmbedsRules() { - for _, subRule := range rule.Rules { - result += fmt.Sprintf("%s%s\n", rule.indent(), subRule.String()) - } - } else { - for _, decl := range rule.Declarations { - result += fmt.Sprintf("%s%s\n", rule.indent(), decl.String()) - } - } - - result += fmt.Sprintf("%s}", rule.indentEndBlock()) - } - - return result -} - -// Returns identation spaces for declarations and rules -func (rule *Rule) indent() string { - result := "" - - for i := 0; i < ((rule.EmbedLevel + 1) * indentSpace); i++ { - result += " " - } - - return result -} - -// Returns identation spaces for end of block character -func (rule *Rule) indentEndBlock() string { - result := "" - - for i := 0; i < (rule.EmbedLevel * indentSpace); i++ { - result += " " - } - - return result -} diff --git a/vendor/github.com/aymerick/douceur/css/stylesheet.go b/vendor/github.com/aymerick/douceur/css/stylesheet.go deleted file mode 100644 index 6b32c2ec..00000000 --- a/vendor/github.com/aymerick/douceur/css/stylesheet.go +++ /dev/null @@ -1,25 +0,0 @@ -package css - -// Stylesheet represents a parsed stylesheet -type Stylesheet struct { - Rules []*Rule -} - -// NewStylesheet instanciate a new Stylesheet -func NewStylesheet() *Stylesheet { - return &Stylesheet{} -} - -// Returns string representation of the Stylesheet -func (sheet *Stylesheet) String() string { - result := "" - - for _, rule := range sheet.Rules { - if result != "" { - result += "\n" - } - result += rule.String() - } - - return result -} diff --git a/vendor/github.com/aymerick/douceur/parser/parser.go b/vendor/github.com/aymerick/douceur/parser/parser.go deleted file mode 100644 index 6c4917cc..00000000 --- a/vendor/github.com/aymerick/douceur/parser/parser.go +++ /dev/null @@ -1,409 +0,0 @@ -package parser - -import ( - "errors" - "fmt" - "regexp" - "strings" - - "github.com/gorilla/css/scanner" - - "github.com/aymerick/douceur/css" -) - -const ( - importantSuffixRegexp = `(?i)\s*!important\s*$` -) - -var ( - importantRegexp *regexp.Regexp -) - -// Parser represents a CSS parser -type Parser struct { - scan *scanner.Scanner // Tokenizer - - // Tokens parsed but not consumed yet - tokens []*scanner.Token - - // Rule embedding level - embedLevel int -} - -func init() { - importantRegexp = regexp.MustCompile(importantSuffixRegexp) -} - -// NewParser instanciates a new parser -func NewParser(txt string) *Parser { - return &Parser{ - scan: scanner.New(txt), - } -} - -// Parse parses a whole stylesheet -func Parse(text string) (*css.Stylesheet, error) { - result, err := NewParser(text).ParseStylesheet() - if err != nil { - return nil, err - } - - return result, nil -} - -// ParseDeclarations parses CSS declarations -func ParseDeclarations(text string) ([]*css.Declaration, error) { - result, err := NewParser(text).ParseDeclarations() - if err != nil { - return nil, err - } - - return result, nil -} - -// ParseStylesheet parses a stylesheet -func (parser *Parser) ParseStylesheet() (*css.Stylesheet, error) { - result := css.NewStylesheet() - - // Parse BOM - if _, err := parser.parseBOM(); err != nil { - return result, err - } - - // Parse list of rules - rules, err := parser.ParseRules() - if err != nil { - return result, err - } - - result.Rules = rules - - return result, nil -} - -// ParseRules parses a list of rules -func (parser *Parser) ParseRules() ([]*css.Rule, error) { - result := []*css.Rule{} - - inBlock := false - if parser.tokenChar("{") { - // parsing a block of rules - inBlock = true - parser.embedLevel++ - - parser.shiftToken() - } - - for parser.tokenParsable() { - if parser.tokenIgnorable() { - parser.shiftToken() - } else if parser.tokenChar("}") { - if !inBlock { - errMsg := fmt.Sprintf("Unexpected } character: %s", parser.nextToken().String()) - return result, errors.New(errMsg) - } - - parser.shiftToken() - parser.embedLevel-- - - // finished - break - } else { - rule, err := parser.ParseRule() - if err != nil { - return result, err - } - - rule.EmbedLevel = parser.embedLevel - result = append(result, rule) - } - } - - return result, parser.err() -} - -// ParseRule parses a rule -func (parser *Parser) ParseRule() (*css.Rule, error) { - if parser.tokenAtKeyword() { - return parser.parseAtRule() - } - - return parser.parseQualifiedRule() -} - -// ParseDeclarations parses a list of declarations -func (parser *Parser) ParseDeclarations() ([]*css.Declaration, error) { - result := []*css.Declaration{} - - if parser.tokenChar("{") { - parser.shiftToken() - } - - for parser.tokenParsable() { - if parser.tokenIgnorable() { - parser.shiftToken() - } else if parser.tokenChar("}") { - // end of block - parser.shiftToken() - break - } else { - declaration, err := parser.ParseDeclaration() - if err != nil { - return result, err - } - - result = append(result, declaration) - } - } - - return result, parser.err() -} - -// ParseDeclaration parses a declaration -func (parser *Parser) ParseDeclaration() (*css.Declaration, error) { - result := css.NewDeclaration() - curValue := "" - - for parser.tokenParsable() { - if parser.tokenChar(":") { - result.Property = strings.TrimSpace(curValue) - curValue = "" - - parser.shiftToken() - } else if parser.tokenChar(";") || parser.tokenChar("}") { - if result.Property == "" { - errMsg := fmt.Sprintf("Unexpected ; character: %s", parser.nextToken().String()) - return result, errors.New(errMsg) - } - - if importantRegexp.MatchString(curValue) { - result.Important = true - curValue = importantRegexp.ReplaceAllString(curValue, "") - } - - result.Value = strings.TrimSpace(curValue) - - if parser.tokenChar(";") { - parser.shiftToken() - } - - // finished - break - } else { - token := parser.shiftToken() - curValue += token.Value - } - } - - // log.Printf("[parsed] Declaration: %s", result.String()) - - return result, parser.err() -} - -// Parse an At Rule -func (parser *Parser) parseAtRule() (*css.Rule, error) { - // parse rule name (eg: "@import") - token := parser.shiftToken() - - result := css.NewRule(css.AtRule) - result.Name = token.Value - - for parser.tokenParsable() { - if parser.tokenChar(";") { - parser.shiftToken() - - // finished - break - } else if parser.tokenChar("{") { - if result.EmbedsRules() { - // parse rules block - rules, err := parser.ParseRules() - if err != nil { - return result, err - } - - result.Rules = rules - } else { - // parse declarations block - declarations, err := parser.ParseDeclarations() - if err != nil { - return result, err - } - - result.Declarations = declarations - } - - // finished - break - } else { - // parse prelude - prelude, err := parser.parsePrelude() - if err != nil { - return result, err - } - - result.Prelude = prelude - } - } - - // log.Printf("[parsed] Rule: %s", result.String()) - - return result, parser.err() -} - -// Parse a Qualified Rule -func (parser *Parser) parseQualifiedRule() (*css.Rule, error) { - result := css.NewRule(css.QualifiedRule) - - for parser.tokenParsable() { - if parser.tokenChar("{") { - if result.Prelude == "" { - errMsg := fmt.Sprintf("Unexpected { character: %s", parser.nextToken().String()) - return result, errors.New(errMsg) - } - - // parse declarations block - declarations, err := parser.ParseDeclarations() - if err != nil { - return result, err - } - - result.Declarations = declarations - - // finished - break - } else { - // parse prelude - prelude, err := parser.parsePrelude() - if err != nil { - return result, err - } - - result.Prelude = prelude - } - } - - result.Selectors = strings.Split(result.Prelude, ",") - for i, sel := range result.Selectors { - result.Selectors[i] = strings.TrimSpace(sel) - } - - // log.Printf("[parsed] Rule: %s", result.String()) - - return result, parser.err() -} - -// Parse Rule prelude -func (parser *Parser) parsePrelude() (string, error) { - result := "" - - for parser.tokenParsable() && !parser.tokenEndOfPrelude() { - token := parser.shiftToken() - result += token.Value - } - - result = strings.TrimSpace(result) - - // log.Printf("[parsed] prelude: %s", result) - - return result, parser.err() -} - -// Parse BOM -func (parser *Parser) parseBOM() (bool, error) { - if parser.nextToken().Type == scanner.TokenBOM { - parser.shiftToken() - return true, nil - } - - return false, parser.err() -} - -// Returns next token without removing it from tokens buffer -func (parser *Parser) nextToken() *scanner.Token { - if len(parser.tokens) == 0 { - // fetch next token - nextToken := parser.scan.Next() - - // log.Printf("[token] %s => %v", nextToken.Type.String(), nextToken.Value) - - // queue it - parser.tokens = append(parser.tokens, nextToken) - } - - return parser.tokens[0] -} - -// Returns next token and remove it from the tokens buffer -func (parser *Parser) shiftToken() *scanner.Token { - var result *scanner.Token - - result, parser.tokens = parser.tokens[0], parser.tokens[1:] - return result -} - -// Returns tokenizer error, or nil if no error -func (parser *Parser) err() error { - if parser.tokenError() { - token := parser.nextToken() - return fmt.Errorf("Tokenizer error: %s", token.String()) - } - - return nil -} - -// Returns true if next token is Error -func (parser *Parser) tokenError() bool { - return parser.nextToken().Type == scanner.TokenError -} - -// Returns true if next token is EOF -func (parser *Parser) tokenEOF() bool { - return parser.nextToken().Type == scanner.TokenEOF -} - -// Returns true if next token is a whitespace -func (parser *Parser) tokenWS() bool { - return parser.nextToken().Type == scanner.TokenS -} - -// Returns true if next token is a comment -func (parser *Parser) tokenComment() bool { - return parser.nextToken().Type == scanner.TokenComment -} - -// Returns true if next token is a CDO or a CDC -func (parser *Parser) tokenCDOorCDC() bool { - switch parser.nextToken().Type { - case scanner.TokenCDO, scanner.TokenCDC: - return true - default: - return false - } -} - -// Returns true if next token is ignorable -func (parser *Parser) tokenIgnorable() bool { - return parser.tokenWS() || parser.tokenComment() || parser.tokenCDOorCDC() -} - -// Returns true if next token is parsable -func (parser *Parser) tokenParsable() bool { - return !parser.tokenEOF() && !parser.tokenError() -} - -// Returns true if next token is an At Rule keyword -func (parser *Parser) tokenAtKeyword() bool { - return parser.nextToken().Type == scanner.TokenAtKeyword -} - -// Returns true if next token is given character -func (parser *Parser) tokenChar(value string) bool { - token := parser.nextToken() - return (token.Type == scanner.TokenChar) && (token.Value == value) -} - -// Returns true if next token marks the end of a prelude -func (parser *Parser) tokenEndOfPrelude() bool { - return parser.tokenChar(";") || parser.tokenChar("{") -} diff --git a/vendor/github.com/didip/tollbooth/v7/.gitignore b/vendor/github.com/didip/tollbooth/v7/.gitignore new file mode 100644 index 00000000..91ea4d71 --- /dev/null +++ b/vendor/github.com/didip/tollbooth/v7/.gitignore @@ -0,0 +1,3 @@ +/debug +/.vscode +/.idea \ No newline at end of file diff --git a/vendor/github.com/didip/tollbooth/v7/.golangci.yml b/vendor/github.com/didip/tollbooth/v7/.golangci.yml new file mode 100644 index 00000000..880786af --- /dev/null +++ b/vendor/github.com/didip/tollbooth/v7/.golangci.yml @@ -0,0 +1,37 @@ +linters: + enable: + - megacheck + - revive + - govet + - unconvert + - megacheck + - structcheck + - gas + - gocyclo + - dupl + - misspell + - unparam + - varcheck + - deadcode + - typecheck + - ineffassign + - varcheck + - stylecheck + - gochecknoinits + - exportloopref + - gocritic + - nakedret + - gosimple + - prealloc + fast: false + disable-all: true + +issues: + exclude-rules: + - path: _test\.go + linters: + - dupl + - text: "Errors unhandled" + linters: + - gosec + exclude-use-default: false diff --git a/vendor/github.com/aymerick/douceur/LICENSE b/vendor/github.com/didip/tollbooth/v7/LICENSE similarity index 88% rename from vendor/github.com/aymerick/douceur/LICENSE rename to vendor/github.com/didip/tollbooth/v7/LICENSE index 6ce87cd3..349ee1c2 100644 --- a/vendor/github.com/aymerick/douceur/LICENSE +++ b/vendor/github.com/didip/tollbooth/v7/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015 Aymerick JEHANNE +Copyright (c) 2015 Didip Kerabat Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -9,14 +9,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/didip/tollbooth/v7/README.md b/vendor/github.com/didip/tollbooth/v7/README.md new file mode 100644 index 00000000..8a45bd89 --- /dev/null +++ b/vendor/github.com/didip/tollbooth/v7/README.md @@ -0,0 +1,184 @@ +[![GoDoc](https://godoc.org/github.com/didip/tollbooth?status.svg)](http://godoc.org/github.com/didip/tollbooth) +[![license](http://img.shields.io/badge/license-MIT-red.svg?style=flat)](https://raw.githubusercontent.com/didip/tollbooth/master/LICENSE) + +## Tollbooth + +This is a generic middleware to rate-limit HTTP requests. + +**NOTE 1:** This library is considered finished. + +**NOTE 2:** Major version changes are backward-incompatible. `v2.0.0` streamlines the ugliness of the old API. + +## Versions + +**v1.0.0:** This version maintains the old API but all the thirdparty modules are moved to their own repo. + +**v2.x.x:** Brand-new API for the sake of code cleanup, thread safety, & auto-expiring data structures. + +**v3.x.x:** Apparently we have been using golang.org/x/time/rate incorrectly. See issue #48. It always limits X number per 1 second. The time duration is not changeable, so it does not make sense to pass TTL to tollbooth. + +**v4.x.x:** Float64 for max requests per second + +**v5.x.x:** go.mod and go.sum + +**v6.x.x:** Replaced `go-cache` with `github.com/go-pkgz/expirable-cache` because `go-cache` leaks goroutines. + +**v7.x.x:** Replaced `time/rate` with `embedded time/rate` so that we can support more rate limit headers. + +## Five Minute Tutorial + +```go +package main + +import ( + "net/http" + + "github.com/didip/tollbooth/v7" +) + +func HelloHandler(w http.ResponseWriter, req *http.Request) { + w.Write([]byte("Hello, World!")) +} + +func main() { + // Create a request limiter per handler. + http.Handle("/", tollbooth.LimitFuncHandler(tollbooth.NewLimiter(1, nil), HelloHandler)) + http.ListenAndServe(":12345", nil) +} +``` + +## Features + +1. Rate-limit by request's remote IP, path, methods, custom headers, & basic auth usernames. + ```go + import ( + "time" + + "github.com/didip/tollbooth/v7" + "github.com/didip/tollbooth/v7/limiter" + ) + + lmt := tollbooth.NewLimiter(1, nil) + + // or create a limiter with expirable token buckets + // This setting means: + // create a 1 request/second limiter and + // every token bucket in it will expire 1 hour after it was initially set. + lmt = tollbooth.NewLimiter(1, &limiter.ExpirableOptions{DefaultExpirationTTL: time.Hour}) + + // Configure list of places to look for IP address. + // By default it's: "RemoteAddr", "X-Forwarded-For", "X-Real-IP" + // If your application is behind a proxy, set "X-Forwarded-For" first. + lmt.SetIPLookups([]string{"RemoteAddr", "X-Forwarded-For", "X-Real-IP"}) + + // Limit only GET and POST requests. + lmt.SetMethods([]string{"GET", "POST"}) + + // Limit based on basic auth usernames. + // You add them on-load, or later as you handle requests. + lmt.SetBasicAuthUsers([]string{"bob", "jane", "didip", "vip"}) + // You can remove them later as well. + lmt.RemoveBasicAuthUsers([]string{"vip"}) + + // Limit request headers containing certain values. + // You add them on-load, or later as you handle requests. + lmt.SetHeader("X-Access-Token", []string{"abc123", "xyz098"}) + // You can remove all entries at once. + lmt.RemoveHeader("X-Access-Token") + // Or remove specific ones. + lmt.RemoveHeaderEntries("X-Access-Token", []string{"limitless-token"}) + + // By the way, the setters are chainable. Example: + lmt.SetIPLookups([]string{"RemoteAddr", "X-Forwarded-For", "X-Real-IP"}). + SetMethods([]string{"GET", "POST"}). + SetBasicAuthUsers([]string{"sansa"}). + SetBasicAuthUsers([]string{"tyrion"}) + ``` + +2. Compose your own middleware by using `LimitByKeys()`. + +3. Header entries and basic auth users can expire over time (to conserve memory). + + ```go + import "time" + + lmt := tollbooth.NewLimiter(1, nil) + + // Set a custom expiration TTL for token bucket. + lmt.SetTokenBucketExpirationTTL(time.Hour) + + // Set a custom expiration TTL for basic auth users. + lmt.SetBasicAuthExpirationTTL(time.Hour) + + // Set a custom expiration TTL for header entries. + lmt.SetHeaderEntryExpirationTTL(time.Hour) + ``` + +4. Upon rejection, the following HTTP response headers are available to users: + + * `X-Rate-Limit-Limit` The maximum request limit. + + * `X-Rate-Limit-Duration` The rate-limiter duration. + + * `X-Rate-Limit-Request-Forwarded-For` The rejected request `X-Forwarded-For`. + + * `X-Rate-Limit-Request-Remote-Addr` The rejected request `RemoteAddr`. + + Upon both success and rejection [RateLimit](https://datatracker.ietf.org/doc/html/draft-ietf-httpapi-ratelimit-headers) headers are sent: + + * `RateLimit-Limit` The maximum request limit within the time window (1s). + + * `RateLimit-Reset` The rate-limiter time window duration in seconds (always 1s). + + * `RateLimit-Remaining` The remaining tokens. + +5. Customize your own message or function when limit is reached. + + ```go + lmt := tollbooth.NewLimiter(1, nil) + + // Set a custom message. + lmt.SetMessage("You have reached maximum request limit.") + + // Set a custom content-type. + lmt.SetMessageContentType("text/plain; charset=utf-8") + + // Set a custom function for rejection. + lmt.SetOnLimitReached(func(w http.ResponseWriter, r *http.Request) { fmt.Println("A request was rejected") }) + ``` + +6. Tollbooth does not require external storage since it uses an algorithm called [Token Bucket](http://en.wikipedia.org/wiki/Token_bucket) [(Go library: golang.org/x/time/rate)](https://godoc.org/golang.org/x/time/rate). + +## Other Web Frameworks + +Sometimes, other frameworks require a little bit of shim to use Tollbooth. These shims below are contributed by the community, so I make no promises on how well they work. The one I am familiar with are: Chi, Gin, and Negroni. + +* [Chi](https://github.com/didip/tollbooth_chi) + +* [Echo](https://github.com/didip/tollbooth_echo) + +* [FastHTTP](https://github.com/didip/tollbooth_fasthttp) + +* [Gin](https://github.com/didip/tollbooth_gin) + +* [GoRestful](https://github.com/didip/tollbooth_gorestful) + +* [HTTPRouter](https://github.com/didip/tollbooth_httprouter) + +* [Iris](https://github.com/didip/tollbooth_iris) + +* [Negroni](https://github.com/didip/tollbooth_negroni) + +## My other Go libraries + +* [Stopwatch](https://github.com/didip/stopwatch): A small library to measure latency of things. Useful if you want to report latency data to Graphite. + +* [LaborUnion](https://github.com/didip/laborunion): A dynamic worker pool library. + +* [Gomet](https://github.com/didip/gomet): Simple HTTP client & server long poll library for Go. Useful for receiving live updates without needing Websocket. + +## Contributions + +Before sending a PR with code changes, please make sure altered code is covered with tests which are passing, and that golangci-lint shows no errors. + +To check the linter output, [install it](https://golangci-lint.run/usage/install/#local-installation) and then run `golangci-lint run` in the root directory of the repository. diff --git a/vendor/github.com/didip/tollbooth/v7/errors/errors.go b/vendor/github.com/didip/tollbooth/v7/errors/errors.go new file mode 100644 index 00000000..149bc5a1 --- /dev/null +++ b/vendor/github.com/didip/tollbooth/v7/errors/errors.go @@ -0,0 +1,15 @@ +// Package errors provide data structure for errors. +package errors + +import "fmt" + +// HTTPError is an error struct that returns both message and status code. +type HTTPError struct { + Message string + StatusCode int +} + +// Error returns error message. +func (httperror *HTTPError) Error() string { + return fmt.Sprintf("%v: %v", httperror.StatusCode, httperror.Message) +} diff --git a/vendor/github.com/didip/tollbooth/v7/internal/time/AUTHORS b/vendor/github.com/didip/tollbooth/v7/internal/time/AUTHORS new file mode 100644 index 00000000..15167cd7 --- /dev/null +++ b/vendor/github.com/didip/tollbooth/v7/internal/time/AUTHORS @@ -0,0 +1,3 @@ +# This source code refers to The Go Authors for copyright purposes. +# The master list of authors is in the main Go distribution, +# visible at http://tip.golang.org/AUTHORS. diff --git a/vendor/github.com/didip/tollbooth/v7/internal/time/CONTRIBUTORS b/vendor/github.com/didip/tollbooth/v7/internal/time/CONTRIBUTORS new file mode 100644 index 00000000..1c4577e9 --- /dev/null +++ b/vendor/github.com/didip/tollbooth/v7/internal/time/CONTRIBUTORS @@ -0,0 +1,3 @@ +# This source code was written by the Go contributors. +# The master list of contributors is in the main Go distribution, +# visible at http://tip.golang.org/CONTRIBUTORS. diff --git a/vendor/golang.org/x/net/LICENSE b/vendor/github.com/didip/tollbooth/v7/internal/time/LICENSE similarity index 100% rename from vendor/golang.org/x/net/LICENSE rename to vendor/github.com/didip/tollbooth/v7/internal/time/LICENSE diff --git a/vendor/golang.org/x/net/PATENTS b/vendor/github.com/didip/tollbooth/v7/internal/time/PATENTS similarity index 100% rename from vendor/golang.org/x/net/PATENTS rename to vendor/github.com/didip/tollbooth/v7/internal/time/PATENTS diff --git a/vendor/github.com/didip/tollbooth/v7/internal/time/rate/rate.go b/vendor/github.com/didip/tollbooth/v7/internal/time/rate/rate.go new file mode 100644 index 00000000..6c3b442d --- /dev/null +++ b/vendor/github.com/didip/tollbooth/v7/internal/time/rate/rate.go @@ -0,0 +1,396 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package rate provides a rate limiter. +package rate + +import ( + "context" + "fmt" + "math" + "sync" + "time" +) + +// Limit defines the maximum frequency of some events. +// Limit is represented as number of events per second. +// A zero Limit allows no events. +type Limit float64 + +// Inf is the infinite rate limit; it allows all events (even if burst is zero). +const Inf = Limit(math.MaxFloat64) + +// Every converts a minimum time interval between events to a Limit. +func Every(interval time.Duration) Limit { + if interval <= 0 { + return Inf + } + return 1 / Limit(interval.Seconds()) +} + +// A Limiter controls how frequently events are allowed to happen. +// It implements a "token bucket" of size b, initially full and refilled +// at rate r tokens per second. +// Informally, in any large enough time interval, the Limiter limits the +// rate to r tokens per second, with a maximum burst size of b events. +// As a special case, if r == Inf (the infinite rate), b is ignored. +// See https://en.wikipedia.org/wiki/Token_bucket for more about token buckets. +// +// The zero value is a valid Limiter, but it will reject all events. +// Use NewLimiter to create non-zero Limiters. +// +// Limiter has three main methods, Allow, Reserve, and Wait. +// Most callers should use Wait. +// +// Each of the three methods consumes a single token. +// They differ in their behavior when no token is available. +// If no token is available, Allow returns false. +// If no token is available, Reserve returns a reservation for a future token +// and the amount of time the caller must wait before using it. +// If no token is available, Wait blocks until one can be obtained +// or its associated context.Context is canceled. +// +// The methods AllowN, ReserveN, and WaitN consume n tokens. +type Limiter struct { + mu sync.Mutex + limit Limit + burst int + tokens float64 + // last is the last time the limiter's tokens field was updated + last time.Time + // lastEvent is the latest time of a rate-limited event (past or future) + lastEvent time.Time +} + +// Limit returns the maximum overall event rate. +func (lim *Limiter) Limit() Limit { + lim.mu.Lock() + defer lim.mu.Unlock() + return lim.limit +} + +// Burst returns the maximum burst size. Burst is the maximum number of tokens +// that can be consumed in a single call to Allow, Reserve, or Wait, so higher +// Burst values allow more events to happen at once. +// A zero Burst allows no events, unless limit == Inf. +func (lim *Limiter) Burst() int { + lim.mu.Lock() + defer lim.mu.Unlock() + return lim.burst +} + +// NewLimiter returns a new Limiter that allows events up to rate r and permits +// bursts of at most b tokens. +func NewLimiter(r Limit, b int) *Limiter { + return &Limiter{ + limit: r, + burst: b, + } +} + +// Allow is shorthand for AllowN(time.Now(), 1). +func (lim *Limiter) Allow() bool { + return lim.AllowN(time.Now(), 1) +} + +// TokensAt returns the number of tokens available for the given time. +func (lim *Limiter) TokensAt(t time.Time) float64 { + lim.mu.Lock() + _, _, tokens := lim.advance(t) // does not mutate lim + lim.mu.Unlock() + return tokens +} + +// AllowN reports whether n events may happen at time now. +// Use this method if you intend to drop / skip events that exceed the rate limit. +// Otherwise use Reserve or Wait. +func (lim *Limiter) AllowN(now time.Time, n int) bool { + return lim.reserveN(now, n, 0).ok +} + +// A Reservation holds information about events that are permitted by a Limiter to happen after a delay. +// A Reservation may be canceled, which may enable the Limiter to permit additional events. +type Reservation struct { + ok bool + lim *Limiter + tokens int + timeToAct time.Time + // This is the Limit at reservation time, it can change later. + limit Limit +} + +// OK returns whether the limiter can provide the requested number of tokens +// within the maximum wait time. If OK is false, Delay returns InfDuration, and +// Cancel does nothing. +func (r *Reservation) OK() bool { + return r.ok +} + +// Delay is shorthand for DelayFrom(time.Now()). +func (r *Reservation) Delay() time.Duration { + return r.DelayFrom(time.Now()) +} + +// InfDuration is the duration returned by Delay when a Reservation is not OK. +const InfDuration = time.Duration(1<<63 - 1) + +// DelayFrom returns the duration for which the reservation holder must wait +// before taking the reserved action. Zero duration means act immediately. +// InfDuration means the limiter cannot grant the tokens requested in this +// Reservation within the maximum wait time. +func (r *Reservation) DelayFrom(now time.Time) time.Duration { + if !r.ok { + return InfDuration + } + delay := r.timeToAct.Sub(now) + if delay < 0 { + return 0 + } + return delay +} + +// Cancel is shorthand for CancelAt(time.Now()). +func (r *Reservation) Cancel() { + r.CancelAt(time.Now()) +} + +// CancelAt indicates that the reservation holder will not perform the reserved action +// and reverses the effects of this Reservation on the rate limit as much as possible, +// considering that other reservations may have already been made. +func (r *Reservation) CancelAt(now time.Time) { + if !r.ok { + return + } + + r.lim.mu.Lock() + defer r.lim.mu.Unlock() + + if r.lim.limit == Inf || r.tokens == 0 || r.timeToAct.Before(now) { + return + } + + // calculate tokens to restore + // The duration between lim.lastEvent and r.timeToAct tells us how many tokens were reserved + // after r was obtained. These tokens should not be restored. + restoreTokens := float64(r.tokens) - r.limit.tokensFromDuration(r.lim.lastEvent.Sub(r.timeToAct)) + if restoreTokens <= 0 { + return + } + // advance time to now + now, _, tokens := r.lim.advance(now) + // calculate new number of tokens + tokens += restoreTokens + if burst := float64(r.lim.burst); tokens > burst { + tokens = burst + } + // update state + r.lim.last = now + r.lim.tokens = tokens + if r.timeToAct == r.lim.lastEvent { + prevEvent := r.timeToAct.Add(r.limit.durationFromTokens(float64(-r.tokens))) + if !prevEvent.Before(now) { + r.lim.lastEvent = prevEvent + } + } +} + +// Reserve is shorthand for ReserveN(time.Now(), 1). +func (lim *Limiter) Reserve() *Reservation { + return lim.ReserveN(time.Now(), 1) +} + +// ReserveN returns a Reservation that indicates how long the caller must wait before n events happen. +// The Limiter takes this Reservation into account when allowing future events. +// The returned Reservation’s OK() method returns false if n exceeds the Limiter's burst size. +// Usage example: +// r := lim.ReserveN(time.Now(), 1) +// if !r.OK() { +// // Not allowed to act! Did you remember to set lim.burst to be > 0 ? +// return +// } +// time.Sleep(r.Delay()) +// Act() +// Use this method if you wish to wait and slow down in accordance with the rate limit without dropping events. +// If you need to respect a deadline or cancel the delay, use Wait instead. +// To drop or skip events exceeding rate limit, use Allow instead. +func (lim *Limiter) ReserveN(now time.Time, n int) *Reservation { + r := lim.reserveN(now, n, InfDuration) + return &r +} + +// Wait is shorthand for WaitN(ctx, 1). +func (lim *Limiter) Wait(ctx context.Context) (err error) { + return lim.WaitN(ctx, 1) +} + +// WaitN blocks until lim permits n events to happen. +// It returns an error if n exceeds the Limiter's burst size, the Context is +// canceled, or the expected wait time exceeds the Context's Deadline. +// The burst limit is ignored if the rate limit is Inf. +func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) { + lim.mu.Lock() + burst := lim.burst + limit := lim.limit + lim.mu.Unlock() + + if n > burst && limit != Inf { + return fmt.Errorf("rate: Wait(n=%d) exceeds limiter's burst %d", n, burst) + } + // Check if ctx is already cancelled + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + // Determine wait limit + now := time.Now() + waitLimit := InfDuration + if deadline, ok := ctx.Deadline(); ok { + waitLimit = deadline.Sub(now) + } + // Reserve + r := lim.reserveN(now, n, waitLimit) + if !r.ok { + return fmt.Errorf("rate: Wait(n=%d) would exceed context deadline", n) + } + // Wait if necessary + delay := r.DelayFrom(now) + if delay == 0 { + return nil + } + t := time.NewTimer(delay) + defer t.Stop() + select { + case <-t.C: + // We can proceed. + return nil + case <-ctx.Done(): + // Context was canceled before we could proceed. Cancel the + // reservation, which may permit other events to proceed sooner. + r.Cancel() + return ctx.Err() + } +} + +// SetLimit is shorthand for SetLimitAt(time.Now(), newLimit). +func (lim *Limiter) SetLimit(newLimit Limit) { + lim.SetLimitAt(time.Now(), newLimit) +} + +// SetLimitAt sets a new Limit for the limiter. The new Limit, and Burst, may be violated +// or underutilized by those which reserved (using Reserve or Wait) but did not yet act +// before SetLimitAt was called. +func (lim *Limiter) SetLimitAt(now time.Time, newLimit Limit) { + lim.mu.Lock() + defer lim.mu.Unlock() + + now, _, tokens := lim.advance(now) + + lim.last = now + lim.tokens = tokens + lim.limit = newLimit +} + +// SetBurst is shorthand for SetBurstAt(time.Now(), newBurst). +func (lim *Limiter) SetBurst(newBurst int) { + lim.SetBurstAt(time.Now(), newBurst) +} + +// SetBurstAt sets a new burst size for the limiter. +func (lim *Limiter) SetBurstAt(now time.Time, newBurst int) { + lim.mu.Lock() + defer lim.mu.Unlock() + + now, _, tokens := lim.advance(now) + + lim.last = now + lim.tokens = tokens + lim.burst = newBurst +} + +// reserveN is a helper method for AllowN, ReserveN, and WaitN. +// maxFutureReserve specifies the maximum reservation wait duration allowed. +// reserveN returns Reservation, not *Reservation, to avoid allocation in AllowN and WaitN. +func (lim *Limiter) reserveN(now time.Time, n int, maxFutureReserve time.Duration) Reservation { + lim.mu.Lock() + + if lim.limit == Inf { + lim.mu.Unlock() + return Reservation{ + ok: true, + lim: lim, + tokens: n, + timeToAct: now, + } + } + + now, last, tokens := lim.advance(now) + + // Calculate the remaining number of tokens resulting from the request. + tokens -= float64(n) + + // Calculate the wait duration + var waitDuration time.Duration + if tokens < 0 { + waitDuration = lim.limit.durationFromTokens(-tokens) + } + + // Decide result + ok := n <= lim.burst && waitDuration <= maxFutureReserve + + // Prepare reservation + r := Reservation{ + ok: ok, + lim: lim, + limit: lim.limit, + } + if ok { + r.tokens = n + r.timeToAct = now.Add(waitDuration) + } + + // Update state + if ok { + lim.last = now + lim.tokens = tokens + lim.lastEvent = r.timeToAct + } else { + lim.last = last + } + + lim.mu.Unlock() + return r +} + +// advance calculates and returns an updated state for lim resulting from the passage of time. +// lim is not changed. +// advance requires that lim.mu is held. +func (lim *Limiter) advance(now time.Time) (newNow time.Time, newLast time.Time, newTokens float64) { + last := lim.last + if now.Before(last) { + last = now + } + + // Calculate the new number of tokens, due to time that passed. + elapsed := now.Sub(last) + delta := lim.limit.tokensFromDuration(elapsed) + tokens := lim.tokens + delta + if burst := float64(lim.burst); tokens > burst { + tokens = burst + } + return now, last, tokens +} + +// durationFromTokens is a unit conversion function from the number of tokens to the duration +// of time it takes to accumulate them at a rate of limit tokens per second. +func (limit Limit) durationFromTokens(tokens float64) time.Duration { + seconds := tokens / float64(limit) + return time.Duration(float64(time.Second) * seconds) +} + +// tokensFromDuration is a unit conversion function from a time duration to the number of tokens +// which could be accumulated during that duration at a rate of limit tokens per second. +func (limit Limit) tokensFromDuration(d time.Duration) float64 { + return d.Seconds() * float64(limit) +} diff --git a/vendor/github.com/didip/tollbooth/v7/libstring/libstring.go b/vendor/github.com/didip/tollbooth/v7/libstring/libstring.go new file mode 100644 index 00000000..730b6549 --- /dev/null +++ b/vendor/github.com/didip/tollbooth/v7/libstring/libstring.go @@ -0,0 +1,101 @@ +// Package libstring provides various string related functions. +package libstring + +import ( + "net" + "net/http" + "strings" +) + +// StringInSlice finds needle in a slice of strings. +func StringInSlice(sliceString []string, needle string) bool { + for _, b := range sliceString { + if b == needle { + return true + } + } + return false +} + +// RemoteIP finds IP Address given http.Request struct. +func RemoteIP(ipLookups []string, forwardedForIndexFromBehind int, r *http.Request) string { + realIP := r.Header.Get("X-Real-IP") + forwardedFor := r.Header.Get("X-Forwarded-For") + + for _, lookup := range ipLookups { + if lookup == "RemoteAddr" { + // 1. Cover the basic use cases for both ipv4 and ipv6 + ip, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + // 2. Upon error, just return the remote addr. + return r.RemoteAddr + } + return ip + } + if lookup == "X-Forwarded-For" && forwardedFor != "" { + // X-Forwarded-For is potentially a list of addresses separated with "," + parts := strings.Split(forwardedFor, ",") + for i, p := range parts { + parts[i] = strings.TrimSpace(p) + } + + partIndex := len(parts) - 1 - forwardedForIndexFromBehind + if partIndex < 0 { + partIndex = 0 + } + + return parts[partIndex] + } + if lookup == "X-Real-IP" && realIP != "" { + return realIP + } + } + + return "" +} + +// CanonicalizeIP returns a form of ip suitable for comparison to other IPs. +// For IPv4 addresses, this is simply the whole string. +// For IPv6 addresses, this is the /64 prefix. +func CanonicalizeIP(ip string) string { + isIPv6 := false + // This is how net.ParseIP decides if an address is IPv6 + // https://cs.opensource.google/go/go/+/refs/tags/go1.17.7:src/net/ip.go;l=704 + for i := 0; !isIPv6 && i < len(ip); i++ { + switch ip[i] { + case '.': + // IPv4 + return ip + case ':': + // IPv6 + isIPv6 = true + } + } + if !isIPv6 { + // Not an IP address at all + return ip + } + + // By default, the string representation of a net.IPNet (masked IP address) is just + // "full_address/mask_bits". But using that will result in different addresses with + // the same /64 prefix comparing differently. So we need to zero out the last 64 bits + // so that all IPs in the same prefix will be the same. + // + // Note: When 1.18 is the minimum Go version, this can be written more cleanly like: + // netip.PrefixFrom(netip.MustParseAddr(ipv6), 64).Masked().Addr().String() + // (With appropriate error checking.) + + ipv6 := net.ParseIP(ip) + if ipv6 == nil { + return ip + } + + const bytesToZero = (128 - 64) / 8 + for i := len(ipv6) - bytesToZero; i < len(ipv6); i++ { + ipv6[i] = 0 + } + + // Note that this doesn't have the "/64" suffix customary with a CIDR representation, + // but those three bytes add nothing for us. + return ipv6.String() +} diff --git a/vendor/github.com/didip/tollbooth/v7/limiter/limiter.go b/vendor/github.com/didip/tollbooth/v7/limiter/limiter.go new file mode 100644 index 00000000..c64a7f26 --- /dev/null +++ b/vendor/github.com/didip/tollbooth/v7/limiter/limiter.go @@ -0,0 +1,610 @@ +// Package limiter provides data structure to configure rate-limiter. +package limiter + +import ( + "net/http" + "sync" + "time" + + cache "github.com/go-pkgz/expirable-cache" + + "github.com/didip/tollbooth/v7/internal/time/rate" +) + +// New is a constructor for Limiter. +func New(generalExpirableOptions *ExpirableOptions) *Limiter { + lmt := &Limiter{} + + lmt.SetMessageContentType("text/plain; charset=utf-8"). + SetMessage("You have reached maximum request limit."). + SetStatusCode(429). + SetOnLimitReached(nil). + SetIPLookups([]string{"RemoteAddr", "X-Forwarded-For", "X-Real-IP"}). + SetForwardedForIndexFromBehind(0). + SetHeaders(make(map[string][]string)). + SetContextValues(make(map[string][]string)). + SetIgnoreURL(false) + + if generalExpirableOptions != nil { + lmt.generalExpirableOptions = generalExpirableOptions + } else { + lmt.generalExpirableOptions = &ExpirableOptions{} + } + + // Default for DefaultExpirationTTL is 10 years. + if lmt.generalExpirableOptions.DefaultExpirationTTL <= 0 { + lmt.generalExpirableOptions.DefaultExpirationTTL = 87600 * time.Hour + } + + lmt.tokenBuckets, _ = cache.NewCache(cache.TTL(lmt.generalExpirableOptions.DefaultExpirationTTL)) + + lmt.basicAuthUsers, _ = cache.NewCache(cache.TTL(lmt.generalExpirableOptions.DefaultExpirationTTL)) + + return lmt +} + +// Limiter is a config struct to limit a particular request handler. +type Limiter struct { + // Maximum number of requests to limit per second. + max float64 + + // Limiter burst size + burst int + + // HTTP message when limit is reached. + message string + + // Content-Type for Message + messageContentType string + + // HTTP status code when limit is reached. + statusCode int + + // A function to call when a request is rejected. + onLimitReached func(w http.ResponseWriter, r *http.Request) + + // An option to write back what you want upon reaching a limit. + overrideDefaultResponseWriter bool + + // List of places to look up IP address. + // Default is "RemoteAddr", "X-Forwarded-For", "X-Real-IP". + // You can rearrange the order as you like. + ipLookups []string + + forwardedForIndex int + + // List of HTTP Methods to limit (GET, POST, PUT, etc.). + // Empty means limit all methods. + methods []string + + // Able to configure token bucket expirations. + generalExpirableOptions *ExpirableOptions + + // List of basic auth usernames to limit. + basicAuthUsers cache.Cache + + // Map of HTTP headers to limit. + // Empty means skip headers checking. + headers map[string]cache.Cache + + // Map of Context values to limit. + contextValues map[string]cache.Cache + + // Map of limiters with TTL + tokenBuckets cache.Cache + + // Ignore URL on the rate limiter keys + ignoreURL bool + + tokenBucketExpirationTTL time.Duration + basicAuthExpirationTTL time.Duration + headerEntryExpirationTTL time.Duration + contextEntryExpirationTTL time.Duration + + sync.RWMutex +} + +// SetTokenBucketExpirationTTL is thread-safe way of setting custom token bucket expiration TTL. +func (l *Limiter) SetTokenBucketExpirationTTL(ttl time.Duration) *Limiter { + l.Lock() + l.tokenBucketExpirationTTL = ttl + l.Unlock() + + return l +} + +// GetTokenBucketExpirationTTL is thread-safe way of getting custom token bucket expiration TTL. +func (l *Limiter) GetTokenBucketExpirationTTL() time.Duration { + l.RLock() + defer l.RUnlock() + return l.tokenBucketExpirationTTL +} + +// SetBasicAuthExpirationTTL is thread-safe way of setting custom basic auth expiration TTL. +func (l *Limiter) SetBasicAuthExpirationTTL(ttl time.Duration) *Limiter { + l.Lock() + l.basicAuthExpirationTTL = ttl + l.Unlock() + + return l +} + +// GetBasicAuthExpirationTTL is thread-safe way of getting custom basic auth expiration TTL. +func (l *Limiter) GetBasicAuthExpirationTTL() time.Duration { + l.RLock() + defer l.RUnlock() + return l.basicAuthExpirationTTL +} + +// SetHeaderEntryExpirationTTL is thread-safe way of setting custom basic auth expiration TTL. +func (l *Limiter) SetHeaderEntryExpirationTTL(ttl time.Duration) *Limiter { + l.Lock() + l.headerEntryExpirationTTL = ttl + l.Unlock() + + return l +} + +// GetHeaderEntryExpirationTTL is thread-safe way of getting custom basic auth expiration TTL. +func (l *Limiter) GetHeaderEntryExpirationTTL() time.Duration { + l.RLock() + defer l.RUnlock() + return l.headerEntryExpirationTTL +} + +// SetContextValueEntryExpirationTTL is thread-safe way of setting custom Context value expiration TTL. +func (l *Limiter) SetContextValueEntryExpirationTTL(ttl time.Duration) *Limiter { + l.Lock() + l.contextEntryExpirationTTL = ttl + l.Unlock() + + return l +} + +// GetContextValueEntryExpirationTTL is thread-safe way of getting custom Context value expiration TTL. +func (l *Limiter) GetContextValueEntryExpirationTTL() time.Duration { + l.RLock() + defer l.RUnlock() + return l.contextEntryExpirationTTL +} + +// SetMax is thread-safe way of setting maximum number of requests to limit per second. +func (l *Limiter) SetMax(max float64) *Limiter { + l.Lock() + l.max = max + l.Unlock() + + return l +} + +// GetMax is thread-safe way of getting maximum number of requests to limit per second. +func (l *Limiter) GetMax() float64 { + l.RLock() + defer l.RUnlock() + return l.max +} + +// SetBurst is thread-safe way of setting maximum burst size. +func (l *Limiter) SetBurst(burst int) *Limiter { + l.Lock() + l.burst = burst + l.Unlock() + + return l +} + +// GetBurst is thread-safe way of setting maximum burst size. +func (l *Limiter) GetBurst() int { + l.RLock() + defer l.RUnlock() + + return l.burst +} + +// SetMessage is thread-safe way of setting HTTP message when limit is reached. +func (l *Limiter) SetMessage(msg string) *Limiter { + l.Lock() + l.message = msg + l.Unlock() + + return l +} + +// GetMessage is thread-safe way of getting HTTP message when limit is reached. +func (l *Limiter) GetMessage() string { + l.RLock() + defer l.RUnlock() + return l.message +} + +// SetMessageContentType is thread-safe way of setting HTTP message Content-Type when limit is reached. +func (l *Limiter) SetMessageContentType(contentType string) *Limiter { + l.Lock() + l.messageContentType = contentType + l.Unlock() + + return l +} + +// GetMessageContentType is thread-safe way of getting HTTP message Content-Type when limit is reached. +func (l *Limiter) GetMessageContentType() string { + l.RLock() + defer l.RUnlock() + return l.messageContentType +} + +// SetStatusCode is thread-safe way of setting HTTP status code when limit is reached. +func (l *Limiter) SetStatusCode(statusCode int) *Limiter { + l.Lock() + l.statusCode = statusCode + l.Unlock() + + return l +} + +// GetStatusCode is thread-safe way of getting HTTP status code when limit is reached. +func (l *Limiter) GetStatusCode() int { + l.RLock() + defer l.RUnlock() + return l.statusCode +} + +// SetOnLimitReached is thread-safe way of setting after-rejection function when limit is reached. +func (l *Limiter) SetOnLimitReached(fn func(w http.ResponseWriter, r *http.Request)) *Limiter { + l.Lock() + l.onLimitReached = fn + l.Unlock() + + return l +} + +// ExecOnLimitReached is thread-safe way of executing after-rejection function when limit is reached. +func (l *Limiter) ExecOnLimitReached(w http.ResponseWriter, r *http.Request) { + l.RLock() + defer l.RUnlock() + + fn := l.onLimitReached + if fn != nil { + fn(w, r) + } +} + +// SetOverrideDefaultResponseWriter is a thread-safe way of setting the response writer override variable. +func (l *Limiter) SetOverrideDefaultResponseWriter(override bool) { + l.Lock() + l.overrideDefaultResponseWriter = override + l.Unlock() +} + +// GetOverrideDefaultResponseWriter is a thread-safe way of getting the response writer override variable. +func (l *Limiter) GetOverrideDefaultResponseWriter() bool { + l.RLock() + defer l.RUnlock() + return l.overrideDefaultResponseWriter +} + +// SetIPLookups is thread-safe way of setting list of places to look up IP address. +func (l *Limiter) SetIPLookups(ipLookups []string) *Limiter { + l.Lock() + l.ipLookups = ipLookups + l.Unlock() + + return l +} + +// GetIPLookups is thread-safe way of getting list of places to look up IP address. +func (l *Limiter) GetIPLookups() []string { + l.RLock() + defer l.RUnlock() + return l.ipLookups +} + +// SetIgnoreURL is thread-safe way of setting whenever ignore the URL on rate limit keys +func (l *Limiter) SetIgnoreURL(enabled bool) *Limiter { + l.Lock() + l.ignoreURL = enabled + l.Unlock() + + return l +} + +// GetIgnoreURL returns whether the URL is ignored in the rate limit key set +func (l *Limiter) GetIgnoreURL() bool { + l.RLock() + defer l.RUnlock() + return l.ignoreURL +} + +// SetForwardedForIndexFromBehind is thread-safe way of setting which X-Forwarded-For index to choose. +func (l *Limiter) SetForwardedForIndexFromBehind(forwardedForIndex int) *Limiter { + l.Lock() + l.forwardedForIndex = forwardedForIndex + l.Unlock() + + return l +} + +// GetForwardedForIndexFromBehind is thread-safe way of getting which X-Forwarded-For index to choose. +func (l *Limiter) GetForwardedForIndexFromBehind() int { + l.RLock() + defer l.RUnlock() + return l.forwardedForIndex +} + +// SetMethods is thread-safe way of setting list of HTTP Methods to limit (GET, POST, PUT, etc.). +func (l *Limiter) SetMethods(methods []string) *Limiter { + l.Lock() + l.methods = methods + l.Unlock() + + return l +} + +// GetMethods is thread-safe way of getting list of HTTP Methods to limit (GET, POST, PUT, etc.). +func (l *Limiter) GetMethods() []string { + l.RLock() + defer l.RUnlock() + return l.methods +} + +// SetBasicAuthUsers is thread-safe way of setting list of basic auth usernames to limit. +func (l *Limiter) SetBasicAuthUsers(basicAuthUsers []string) *Limiter { + ttl := l.GetBasicAuthExpirationTTL() + if ttl <= 0 { + ttl = l.generalExpirableOptions.DefaultExpirationTTL + } + + for _, basicAuthUser := range basicAuthUsers { + l.basicAuthUsers.Set(basicAuthUser, true, ttl) + } + + return l +} + +// GetBasicAuthUsers is thread-safe way of getting list of basic auth usernames to limit. +func (l *Limiter) GetBasicAuthUsers() []string { + return l.basicAuthUsers.Keys() +} + +// RemoveBasicAuthUsers is thread-safe way of removing basic auth usernames from existing list. +func (l *Limiter) RemoveBasicAuthUsers(basicAuthUsers []string) *Limiter { + for _, toBeRemoved := range basicAuthUsers { + l.basicAuthUsers.Invalidate(toBeRemoved) + } + + return l +} + +// DeleteExpiredTokenBuckets is thread-safe way of deleting expired token buckets +func (l *Limiter) DeleteExpiredTokenBuckets() { + l.tokenBuckets.DeleteExpired() +} + +// SetHeaders is thread-safe way of setting map of HTTP headers to limit. +func (l *Limiter) SetHeaders(headers map[string][]string) *Limiter { + if l.headers == nil { + l.headers = make(map[string]cache.Cache) + } + + for header, entries := range headers { + l.SetHeader(header, entries) + } + + return l +} + +// GetHeaders is thread-safe way of getting map of HTTP headers to limit. +func (l *Limiter) GetHeaders() map[string][]string { + results := make(map[string][]string) + + l.RLock() + defer l.RUnlock() + + for header, entriesAsGoCache := range l.headers { + results[header] = entriesAsGoCache.Keys() + } + + return results +} + +// SetHeader is thread-safe way of setting entries of 1 HTTP header. +func (l *Limiter) SetHeader(header string, entries []string) *Limiter { + l.RLock() + existing, found := l.headers[header] + l.RUnlock() + + ttl := l.GetHeaderEntryExpirationTTL() + if ttl <= 0 { + ttl = l.generalExpirableOptions.DefaultExpirationTTL + } + + if !found { + existing, _ = cache.NewCache(cache.TTL(ttl)) + } + + for _, entry := range entries { + existing.Set(entry, true, ttl) + } + + l.Lock() + l.headers[header] = existing + l.Unlock() + + return l +} + +// GetHeader is thread-safe way of getting entries of 1 HTTP header. +func (l *Limiter) GetHeader(header string) []string { + l.RLock() + entriesAsGoCache := l.headers[header] + l.RUnlock() + + return entriesAsGoCache.Keys() +} + +// RemoveHeader is thread-safe way of removing entries of 1 HTTP header. +func (l *Limiter) RemoveHeader(header string) *Limiter { + ttl := l.GetHeaderEntryExpirationTTL() + if ttl <= 0 { + ttl = l.generalExpirableOptions.DefaultExpirationTTL + } + + l.Lock() + l.headers[header], _ = cache.NewCache(cache.TTL(ttl)) + l.Unlock() + + return l +} + +// RemoveHeaderEntries is thread-safe way of removing new entries to 1 HTTP header rule. +func (l *Limiter) RemoveHeaderEntries(header string, entriesForRemoval []string) *Limiter { + l.RLock() + entries, found := l.headers[header] + l.RUnlock() + + if !found { + return l + } + + for _, toBeRemoved := range entriesForRemoval { + entries.Invalidate(toBeRemoved) + } + + return l +} + +// SetContextValues is thread-safe way of setting map of HTTP headers to limit. +func (l *Limiter) SetContextValues(contextValues map[string][]string) *Limiter { + if l.contextValues == nil { + l.contextValues = make(map[string]cache.Cache) + } + + for contextValue, entries := range contextValues { + l.SetContextValue(contextValue, entries) + } + + return l +} + +// GetContextValues is thread-safe way of getting a map of Context values to limit. +func (l *Limiter) GetContextValues() map[string][]string { + results := make(map[string][]string) + + l.RLock() + defer l.RUnlock() + + for contextValue, entriesAsGoCache := range l.contextValues { + results[contextValue] = entriesAsGoCache.Keys() + } + + return results +} + +// SetContextValue is thread-safe way of setting entries of 1 Context value. +func (l *Limiter) SetContextValue(contextValue string, entries []string) *Limiter { + l.RLock() + existing, found := l.contextValues[contextValue] + l.RUnlock() + + ttl := l.GetContextValueEntryExpirationTTL() + if ttl <= 0 { + ttl = l.generalExpirableOptions.DefaultExpirationTTL + } + + if !found { + existing, _ = cache.NewCache(cache.TTL(ttl)) + } + + for _, entry := range entries { + existing.Set(entry, true, ttl) + } + + l.Lock() + l.contextValues[contextValue] = existing + l.Unlock() + + return l +} + +// GetContextValue is thread-safe way of getting 1 Context value entry. +func (l *Limiter) GetContextValue(contextValue string) []string { + l.RLock() + entriesAsGoCache := l.contextValues[contextValue] + l.RUnlock() + + return entriesAsGoCache.Keys() +} + +// RemoveContextValue is thread-safe way of removing entries of 1 Context value. +func (l *Limiter) RemoveContextValue(contextValue string) *Limiter { + ttl := l.GetContextValueEntryExpirationTTL() + if ttl <= 0 { + ttl = l.generalExpirableOptions.DefaultExpirationTTL + } + + l.Lock() + l.contextValues[contextValue], _ = cache.NewCache(cache.TTL(ttl)) + l.Unlock() + + return l +} + +// RemoveContextValuesEntries is thread-safe way of removing entries to a ContextValue. +func (l *Limiter) RemoveContextValuesEntries(contextValue string, entriesForRemoval []string) *Limiter { + l.RLock() + entries, found := l.contextValues[contextValue] + l.RUnlock() + + if !found { + return l + } + + for _, toBeRemoved := range entriesForRemoval { + entries.Invalidate(toBeRemoved) + } + + return l +} + +func (l *Limiter) limitReachedWithTokenBucketTTL(key string, tokenBucketTTL time.Duration) bool { + lmtMax := l.GetMax() + lmtBurst := l.GetBurst() + l.Lock() + defer l.Unlock() + + if _, found := l.tokenBuckets.Get(key); !found { + l.tokenBuckets.Set( + key, + rate.NewLimiter(rate.Limit(lmtMax), lmtBurst), + tokenBucketTTL, + ) + } + + expiringMap, found := l.tokenBuckets.Get(key) + if !found { + return false + } + + return !expiringMap.(*rate.Limiter).Allow() +} + +// LimitReached returns a bool indicating if the Bucket identified by key ran out of tokens. +func (l *Limiter) LimitReached(key string) bool { + ttl := l.GetTokenBucketExpirationTTL() + + if ttl <= 0 { + ttl = l.generalExpirableOptions.DefaultExpirationTTL + } + + return l.limitReachedWithTokenBucketTTL(key, ttl) +} + +// Tokens returns current amount of tokens left in the Bucket identified by key. +func (l *Limiter) Tokens(key string) int { + expiringMap, found := l.tokenBuckets.Get(key) + if !found { + return 0 + } + + return int(expiringMap.(*rate.Limiter).TokensAt(time.Now())) +} diff --git a/vendor/github.com/didip/tollbooth/v7/limiter/limiter_options.go b/vendor/github.com/didip/tollbooth/v7/limiter/limiter_options.go new file mode 100644 index 00000000..e5f537b5 --- /dev/null +++ b/vendor/github.com/didip/tollbooth/v7/limiter/limiter_options.go @@ -0,0 +1,14 @@ +package limiter + +import ( + "time" +) + +// ExpirableOptions are options used for new limiter creation +type ExpirableOptions struct { + DefaultExpirationTTL time.Duration + + // How frequently expire job triggers + // Deprecated: not used anymore + ExpireJobInterval time.Duration +} diff --git a/vendor/github.com/didip/tollbooth/v7/tollbooth.go b/vendor/github.com/didip/tollbooth/v7/tollbooth.go new file mode 100644 index 00000000..0dcf82e0 --- /dev/null +++ b/vendor/github.com/didip/tollbooth/v7/tollbooth.go @@ -0,0 +1,349 @@ +// Package tollbooth provides rate-limiting logic to HTTP request handler. +package tollbooth + +import ( + "fmt" + "math" + "net/http" + "strings" + + "github.com/didip/tollbooth/v7/errors" + "github.com/didip/tollbooth/v7/libstring" + "github.com/didip/tollbooth/v7/limiter" +) + +// setResponseHeaders configures X-Rate-Limit-Limit and X-Rate-Limit-Duration +func setResponseHeaders(lmt *limiter.Limiter, w http.ResponseWriter, r *http.Request) { + w.Header().Add("X-Rate-Limit-Limit", fmt.Sprintf("%.2f", lmt.GetMax())) + w.Header().Add("X-Rate-Limit-Duration", "1") + + xForwardedFor := r.Header.Get("X-Forwarded-For") + if strings.TrimSpace(xForwardedFor) != "" { + w.Header().Add("X-Rate-Limit-Request-Forwarded-For", xForwardedFor) + } + + w.Header().Add("X-Rate-Limit-Request-Remote-Addr", r.RemoteAddr) +} + +// setRateLimitResponseHeaders configures RateLimit-Limit, RateLimit-Remaining and RateLimit-Reset +// as seen at https://datatracker.ietf.org/doc/html/draft-ietf-httpapi-ratelimit-headers +func setRateLimitResponseHeaders(lmt *limiter.Limiter, w http.ResponseWriter, tokensLeft int) { + w.Header().Add("RateLimit-Limit", fmt.Sprintf("%d", int(math.Round(lmt.GetMax())))) + w.Header().Add("RateLimit-Reset", "1") + w.Header().Add("RateLimit-Remaining", fmt.Sprintf("%d", tokensLeft)) +} + +// NewLimiter is a convenience function to limiter.New. +func NewLimiter(max float64, tbOptions *limiter.ExpirableOptions) *limiter.Limiter { + return limiter.New(tbOptions). + SetMax(max). + SetBurst(int(math.Max(1, max))). + SetIPLookups([]string{"X-Forwarded-For", "X-Real-IP", "RemoteAddr"}) +} + +// LimitByKeys keeps track number of request made by keys separated by pipe. +// It returns HTTPError when limit is exceeded. +func LimitByKeys(lmt *limiter.Limiter, keys []string) *errors.HTTPError { + err, _ := LimitByKeysAndReturn(lmt, keys) + return err +} + +// LimitByKeysAndReturn keeps track number of request made by keys separated by pipe. +// It returns HTTPError when limit is exceeded, and also returns the current limit value. +func LimitByKeysAndReturn(lmt *limiter.Limiter, keys []string) (*errors.HTTPError, int) { + if lmt.LimitReached(strings.Join(keys, "|")) { + return &errors.HTTPError{Message: lmt.GetMessage(), StatusCode: lmt.GetStatusCode()}, 0 + } + + return nil, lmt.Tokens(strings.Join(keys, "|")) +} + +// ShouldSkipLimiter is a series of filter that decides if request should be limited or not. +func ShouldSkipLimiter(lmt *limiter.Limiter, r *http.Request) bool { + // --------------------------------- + // Filter by remote ip + // If we are unable to find remoteIP, skip limiter + remoteIP := libstring.RemoteIP(lmt.GetIPLookups(), lmt.GetForwardedForIndexFromBehind(), r) + remoteIP = libstring.CanonicalizeIP(remoteIP) + if remoteIP == "" { + return true + } + + // --------------------------------- + // Filter by request method + lmtMethods := lmt.GetMethods() + lmtMethodsIsSet := len(lmtMethods) > 0 + + if lmtMethodsIsSet { + // If request does not contain all of the methods in limiter, + // skip limiter + requestMethodDefinedInLimiter := libstring.StringInSlice(lmtMethods, r.Method) + + if !requestMethodDefinedInLimiter { + return true + } + } + + // --------------------------------- + // Filter by request headers + lmtHeaders := lmt.GetHeaders() + lmtHeadersIsSet := len(lmtHeaders) > 0 + + if lmtHeadersIsSet { + // If request does not contain all of the headers in limiter, + // skip limiter + requestHeadersDefinedInLimiter := false + + for headerKey := range lmtHeaders { + reqHeaderValue := r.Header.Get(headerKey) + if reqHeaderValue != "" { + requestHeadersDefinedInLimiter = true + break + } + } + + if !requestHeadersDefinedInLimiter { + return true + } + + // ------------------------------ + // If request contains the header key but not the values, + // skip limiter + requestHeadersDefinedInLimiter = false + + for headerKey, headerValues := range lmtHeaders { + if len(headerValues) == 0 { + requestHeadersDefinedInLimiter = true + continue + } + for _, headerValue := range headerValues { + if r.Header.Get(headerKey) == headerValue { + requestHeadersDefinedInLimiter = true + break + } + } + } + + if !requestHeadersDefinedInLimiter { + return true + } + } + + // --------------------------------- + // Filter by context values + lmtContextValues := lmt.GetContextValues() + lmtContextValuesIsSet := len(lmtContextValues) > 0 + + if lmtContextValuesIsSet { + // If request does not contain all of the contexts in limiter, + // skip limiter + requestContextValuesDefinedInLimiter := false + + for contextKey := range lmtContextValues { + reqContextValue := fmt.Sprintf("%v", r.Context().Value(contextKey)) + if reqContextValue != "" { + requestContextValuesDefinedInLimiter = true + break + } + } + + if !requestContextValuesDefinedInLimiter { + return true + } + + // ------------------------------ + // If request contains the context key but not the values, + // skip limiter + requestContextValuesDefinedInLimiter = false + + for contextKey, contextValues := range lmtContextValues { + for _, contextValue := range contextValues { + if r.Header.Get(contextKey) == contextValue { + requestContextValuesDefinedInLimiter = true + break + } + } + } + + if !requestContextValuesDefinedInLimiter { + return true + } + } + + // --------------------------------- + // Filter by basic auth usernames + lmtBasicAuthUsers := lmt.GetBasicAuthUsers() + lmtBasicAuthUsersIsSet := len(lmtBasicAuthUsers) > 0 + + if lmtBasicAuthUsersIsSet { + // If request does not contain all of the basic auth users in limiter, + // skip limiter + requestAuthUsernameDefinedInLimiter := false + + username, _, ok := r.BasicAuth() + if ok && libstring.StringInSlice(lmtBasicAuthUsers, username) { + requestAuthUsernameDefinedInLimiter = true + } + + if !requestAuthUsernameDefinedInLimiter { + return true + } + } + + return false +} + +// BuildKeys generates a slice of keys to rate-limit by given limiter and request structs. +func BuildKeys(lmt *limiter.Limiter, r *http.Request) [][]string { + remoteIP := libstring.RemoteIP(lmt.GetIPLookups(), lmt.GetForwardedForIndexFromBehind(), r) + remoteIP = libstring.CanonicalizeIP(remoteIP) + path := r.URL.Path + sliceKeys := make([][]string, 0) + + lmtMethods := lmt.GetMethods() + lmtHeaders := lmt.GetHeaders() + lmtContextValues := lmt.GetContextValues() + lmtBasicAuthUsers := lmt.GetBasicAuthUsers() + lmtIgnoreURL := lmt.GetIgnoreURL() + + lmtHeadersIsSet := len(lmtHeaders) > 0 + lmtContextValuesIsSet := len(lmtContextValues) > 0 + lmtBasicAuthUsersIsSet := len(lmtBasicAuthUsers) > 0 + + usernameToLimit := "" + if lmtBasicAuthUsersIsSet { + username, _, ok := r.BasicAuth() + if ok && libstring.StringInSlice(lmtBasicAuthUsers, username) { + usernameToLimit = username + } + } + + headerValuesToLimit := [][]string{} + if lmtHeadersIsSet { + for headerKey, headerValues := range lmtHeaders { + reqHeaderValue := r.Header.Get(headerKey) + if reqHeaderValue == "" { + continue + } + + if len(headerValues) == 0 { + // If header values are empty, rate-limit all request containing headerKey. + headerValuesToLimit = append(headerValuesToLimit, []string{headerKey, reqHeaderValue}) + + } else { + // If header values are not empty, rate-limit all request with headerKey and headerValues. + for _, headerValue := range headerValues { + if r.Header.Get(headerKey) == headerValue { + headerValuesToLimit = append(headerValuesToLimit, []string{headerKey, headerValue}) + break + } + } + } + } + } + + contextValuesToLimit := [][]string{} + if lmtContextValuesIsSet { + for contextKey, contextValues := range lmtContextValues { + reqContextValue := fmt.Sprintf("%v", r.Context().Value(contextKey)) + if reqContextValue == "" { + continue + } + + if len(contextValues) == 0 { + // If context values are empty, rate-limit all request containing contextKey. + contextValuesToLimit = append(contextValuesToLimit, []string{contextKey, reqContextValue}) + + } else { + // If context values are not empty, rate-limit all request with contextKey and contextValues. + for _, contextValue := range contextValues { + if reqContextValue == contextValue { + contextValuesToLimit = append(contextValuesToLimit, []string{contextKey, contextValue}) + break + } + } + } + } + } + + sliceKey := []string{remoteIP} + if !lmtIgnoreURL { + sliceKey = append(sliceKey, path) + } + + sliceKey = append(sliceKey, lmtMethods...) + + for _, header := range headerValuesToLimit { + sliceKey = append(sliceKey, header[0], header[1]) + } + + for _, contextValue := range contextValuesToLimit { + sliceKey = append(sliceKey, contextValue[0], contextValue[1]) + } + + sliceKey = append(sliceKey, usernameToLimit) + + sliceKeys = append(sliceKeys, sliceKey) + + return sliceKeys +} + +// LimitByRequest builds keys based on http.Request struct, +// loops through all the keys, and check if any one of them returns HTTPError. +func LimitByRequest(lmt *limiter.Limiter, w http.ResponseWriter, r *http.Request) *errors.HTTPError { + setResponseHeaders(lmt, w, r) + + shouldSkip := ShouldSkipLimiter(lmt, r) + if shouldSkip { + return nil + } + + sliceKeys := BuildKeys(lmt, r) + + // Get the lowest value over all keys to return in headers. + // Start with high arbitrary number so that any limit returned would be lower and would + // overwrite the value we start with. + var tokensLeft = math.MaxInt32 + + // Loop sliceKeys and check if one of them has error. + for _, keys := range sliceKeys { + httpError, keysLimit := LimitByKeysAndReturn(lmt, keys) + if tokensLeft > keysLimit { + tokensLeft = keysLimit + } + if httpError != nil { + setRateLimitResponseHeaders(lmt, w, tokensLeft) + return httpError + } + } + + setRateLimitResponseHeaders(lmt, w, tokensLeft) + return nil +} + +// LimitHandler is a middleware that performs rate-limiting given http.Handler struct. +func LimitHandler(lmt *limiter.Limiter, next http.Handler) http.Handler { + middle := func(w http.ResponseWriter, r *http.Request) { + httpError := LimitByRequest(lmt, w, r) + if httpError != nil { + lmt.ExecOnLimitReached(w, r) + if lmt.GetOverrideDefaultResponseWriter() { + return + } + w.Header().Add("Content-Type", lmt.GetMessageContentType()) + w.WriteHeader(httpError.StatusCode) + w.Write([]byte(httpError.Message)) + return + } + + // There's no rate-limit error, serve the next handler. + next.ServeHTTP(w, r) + } + + return http.HandlerFunc(middle) +} + +// LimitFuncHandler is a middleware that performs rate-limiting given request handler function. +func LimitFuncHandler(lmt *limiter.Limiter, nextFunc func(http.ResponseWriter, *http.Request)) http.Handler { + return LimitHandler(lmt, http.HandlerFunc(nextFunc)) +} diff --git a/vendor/github.com/didip/tollbooth_chi/README.md b/vendor/github.com/didip/tollbooth_chi/README.md new file mode 100644 index 00000000..d1ba02b1 --- /dev/null +++ b/vendor/github.com/didip/tollbooth_chi/README.md @@ -0,0 +1,33 @@ +## tollbooth_chi + +[Chi](https://github.com/pressly/chi) middleware for rate limiting HTTP requests. + + +## Five Minutes Tutorial + +``` +package main + +import ( + "github.com/didip/tollbooth" + "github.com/didip/tollbooth_chi" + "github.com/pressly/chi" + "net/http" + "time" +) + +func main() { + // Create a limiter struct. + limiter := tollbooth.NewLimiter(1, nil) + + r := chi.NewRouter() + + r.Use(tollbooth_chi.LimitHandler(limiter)) + + r.Get("/", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("Hello, world!")) + }) + + http.ListenAndServe(":12345", r) +} +``` diff --git a/vendor/github.com/didip/tollbooth_chi/tollbooth_chi.go b/vendor/github.com/didip/tollbooth_chi/tollbooth_chi.go new file mode 100644 index 00000000..7e45c9a6 --- /dev/null +++ b/vendor/github.com/didip/tollbooth_chi/tollbooth_chi.go @@ -0,0 +1,45 @@ +package tollbooth_chi + +import ( + "net/http" + + "github.com/didip/tollbooth/v7" + "github.com/didip/tollbooth/v7/limiter" +) + +func LimitHandler(lmt *limiter.Limiter) func(http.Handler) http.Handler { + return func(handler http.Handler) http.Handler { + wrapper := &limiterWrapper{ + lmt: lmt, + } + + wrapper.handler = handler + return wrapper + } +} + +type limiterWrapper struct { + lmt *limiter.Limiter + handler http.Handler +} + +func (l *limiterWrapper) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + select { + case <-ctx.Done(): + http.Error(w, "Context was canceled", http.StatusServiceUnavailable) + return + + default: + httpError := tollbooth.LimitByRequest(l.lmt, w, r) + if httpError != nil { + l.lmt.ExecOnLimitReached(w, r) + w.Header().Add("Content-Type", l.lmt.GetMessageContentType()) + w.WriteHeader(httpError.StatusCode) + w.Write([]byte(httpError.Message)) + return + } + + l.handler.ServeHTTP(w, r) + } +} diff --git a/vendor/github.com/go-chi/chi/.gitignore b/vendor/github.com/go-chi/chi/.gitignore new file mode 100644 index 00000000..ba22c99a --- /dev/null +++ b/vendor/github.com/go-chi/chi/.gitignore @@ -0,0 +1,3 @@ +.idea +*.sw? +.vscode diff --git a/vendor/github.com/go-chi/chi/CHANGELOG.md b/vendor/github.com/go-chi/chi/CHANGELOG.md new file mode 100644 index 00000000..7dd07915 --- /dev/null +++ b/vendor/github.com/go-chi/chi/CHANGELOG.md @@ -0,0 +1,269 @@ +# Changelog + +## v1.5.4 (2021-02-27) + +- Undo prior retraction in v1.5.3 as we prepare for v5.0.0 release +- History of changes: see https://github.com/go-chi/chi/compare/v1.5.3...v1.5.4 + + +## v1.5.3 (2021-02-21) + +- Update go.mod to go 1.16 with new retract directive marking all versions without prior go.mod support +- History of changes: see https://github.com/go-chi/chi/compare/v1.5.2...v1.5.3 + + +## v1.5.2 (2021-02-10) + +- Reverting allocation optimization as a precaution as go test -race fails. +- Minor improvements, see history below +- History of changes: see https://github.com/go-chi/chi/compare/v1.5.1...v1.5.2 + + +## v1.5.1 (2020-12-06) + +- Performance improvement: removing 1 allocation by foregoing context.WithValue, thank you @bouk for + your contribution (https://github.com/go-chi/chi/pull/555). Note: new benchmarks posted in README. +- `middleware.CleanPath`: new middleware that clean's request path of double slashes +- deprecate & remove `chi.ServerBaseContext` in favour of stdlib `http.Server#BaseContext` +- plus other tiny improvements, see full commit history below +- History of changes: see https://github.com/go-chi/chi/compare/v4.1.2...v1.5.1 + + +## v1.5.0 (2020-11-12) - now with go.mod support + +`chi` dates back to 2016 with it's original implementation as one of the first routers to adopt the newly introduced +context.Context api to the stdlib -- set out to design a router that is faster, more modular and simpler than anything +else out there -- while not introducing any custom handler types or dependencies. Today, `chi` still has zero dependencies, +and in many ways is future proofed from changes, given it's minimal nature. Between versions, chi's iterations have been very +incremental, with the architecture and api being the same today as it was originally designed in 2016. For this reason it +makes chi a pretty easy project to maintain, as well thanks to the many amazing community contributions over the years +to who all help make chi better (total of 86 contributors to date -- thanks all!). + +Chi has been an labour of love, art and engineering, with the goals to offer beautiful ergonomics, flexibility, performance +and simplicity when building HTTP services with Go. I've strived to keep the router very minimal in surface area / code size, +and always improving the code wherever possible -- and as of today the `chi` package is just 1082 lines of code (not counting +middlewares, which are all optional). As well, I don't have the exact metrics, but from my analysis and email exchanges from +companies and developers, chi is used by thousands of projects around the world -- thank you all as there is no better form of +joy for me than to have art I had started be helpful and enjoyed by others. And of course I use chi in all of my own projects too :) + +For me, the asthetics of chi's code and usage are very important. With the introduction of Go's module support +(which I'm a big fan of), chi's past versioning scheme choice to v2, v3 and v4 would mean I'd require the import path +of "github.com/go-chi/chi/v4", leading to the lengthy discussion at https://github.com/go-chi/chi/issues/462. +Haha, to some, you may be scratching your head why I've spent > 1 year stalling to adopt "/vXX" convention in the import +path -- which isn't horrible in general -- but for chi, I'm unable to accept it as I strive for perfection in it's API design, +aesthetics and simplicity. It just doesn't feel good to me given chi's simple nature -- I do not foresee a "v5" or "v6", +and upgrading between versions in the future will also be just incremental. + +I do understand versioning is a part of the API design as well, which is why the solution for a while has been to "do nothing", +as Go supports both old and new import paths with/out go.mod. However, now that Go module support has had time to iron out kinks and +is adopted everywhere, it's time for chi to get with the times. Luckily, I've discovered a path forward that will make me happy, +while also not breaking anyone's app who adopted a prior versioning from tags in v2/v3/v4. I've made an experimental release of +v1.5.0 with go.mod silently, and tested it with new and old projects, to ensure the developer experience is preserved, and it's +largely unnoticed. Fortunately, Go's toolchain will check the tags of a repo and consider the "latest" tag the one with go.mod. +However, you can still request a specific older tag such as v4.1.2, and everything will "just work". But new users can just +`go get github.com/go-chi/chi` or `go get github.com/go-chi/chi@latest` and they will get the latest version which contains +go.mod support, which is v1.5.0+. `chi` will not change very much over the years, just like it hasn't changed much from 4 years ago. +Therefore, we will stay on v1.x from here on, starting from v1.5.0. Any breaking changes will bump a "minor" release and +backwards-compatible improvements/fixes will bump a "tiny" release. + +For existing projects who want to upgrade to the latest go.mod version, run: `go get -u github.com/go-chi/chi@v1.5.0`, +which will get you on the go.mod version line (as Go's mod cache may still remember v4.x). Brand new systems can run +`go get -u github.com/go-chi/chi` or `go get -u github.com/go-chi/chi@latest` to install chi, which will install v1.5.0+ +built with go.mod support. + +My apologies to the developers who will disagree with the decisions above, but, hope you'll try it and see it's a very +minor request which is backwards compatible and won't break your existing installations. + +Cheers all, happy coding! + + +--- + + +## v4.1.2 (2020-06-02) + +- fix that handles MethodNotAllowed with path variables, thank you @caseyhadden for your contribution +- fix to replace nested wildcards correctly in RoutePattern, thank you @@unmultimedio for your contribution +- History of changes: see https://github.com/go-chi/chi/compare/v4.1.1...v4.1.2 + + +## v4.1.1 (2020-04-16) + +- fix for issue https://github.com/go-chi/chi/issues/411 which allows for overlapping regexp + route to the correct handler through a recursive tree search, thanks to @Jahaja for the PR/fix! +- new middleware.RouteHeaders as a simple router for request headers with wildcard support +- History of changes: see https://github.com/go-chi/chi/compare/v4.1.0...v4.1.1 + + +## v4.1.0 (2020-04-1) + +- middleware.LogEntry: Write method on interface now passes the response header + and an extra interface type useful for custom logger implementations. +- middleware.WrapResponseWriter: minor fix +- middleware.Recoverer: a bit prettier +- History of changes: see https://github.com/go-chi/chi/compare/v4.0.4...v4.1.0 + +## v4.0.4 (2020-03-24) + +- middleware.Recoverer: new pretty stack trace printing (https://github.com/go-chi/chi/pull/496) +- a few minor improvements and fixes +- History of changes: see https://github.com/go-chi/chi/compare/v4.0.3...v4.0.4 + + +## v4.0.3 (2020-01-09) + +- core: fix regexp routing to include default value when param is not matched +- middleware: rewrite of middleware.Compress +- middleware: suppress http.ErrAbortHandler in middleware.Recoverer +- History of changes: see https://github.com/go-chi/chi/compare/v4.0.2...v4.0.3 + + +## v4.0.2 (2019-02-26) + +- Minor fixes +- History of changes: see https://github.com/go-chi/chi/compare/v4.0.1...v4.0.2 + + +## v4.0.1 (2019-01-21) + +- Fixes issue with compress middleware: #382 #385 +- History of changes: see https://github.com/go-chi/chi/compare/v4.0.0...v4.0.1 + + +## v4.0.0 (2019-01-10) + +- chi v4 requires Go 1.10.3+ (or Go 1.9.7+) - we have deprecated support for Go 1.7 and 1.8 +- router: respond with 404 on router with no routes (#362) +- router: additional check to ensure wildcard is at the end of a url pattern (#333) +- middleware: deprecate use of http.CloseNotifier (#347) +- middleware: fix RedirectSlashes to include query params on redirect (#334) +- History of changes: see https://github.com/go-chi/chi/compare/v3.3.4...v4.0.0 + + +## v3.3.4 (2019-01-07) + +- Minor middleware improvements. No changes to core library/router. Moving v3 into its +- own branch as a version of chi for Go 1.7, 1.8, 1.9, 1.10, 1.11 +- History of changes: see https://github.com/go-chi/chi/compare/v3.3.3...v3.3.4 + + +## v3.3.3 (2018-08-27) + +- Minor release +- See https://github.com/go-chi/chi/compare/v3.3.2...v3.3.3 + + +## v3.3.2 (2017-12-22) + +- Support to route trailing slashes on mounted sub-routers (#281) +- middleware: new `ContentCharset` to check matching charsets. Thank you + @csucu for your community contribution! + + +## v3.3.1 (2017-11-20) + +- middleware: new `AllowContentType` handler for explicit whitelist of accepted request Content-Types +- middleware: new `SetHeader` handler for short-hand middleware to set a response header key/value +- Minor bug fixes + + +## v3.3.0 (2017-10-10) + +- New chi.RegisterMethod(method) to add support for custom HTTP methods, see _examples/custom-method for usage +- Deprecated LINK and UNLINK methods from the default list, please use `chi.RegisterMethod("LINK")` and `chi.RegisterMethod("UNLINK")` in an `init()` function + + +## v3.2.1 (2017-08-31) + +- Add new `Match(rctx *Context, method, path string) bool` method to `Routes` interface + and `Mux`. Match searches the mux's routing tree for a handler that matches the method/path +- Add new `RouteMethod` to `*Context` +- Add new `Routes` pointer to `*Context` +- Add new `middleware.GetHead` to route missing HEAD requests to GET handler +- Updated benchmarks (see README) + + +## v3.1.5 (2017-08-02) + +- Setup golint and go vet for the project +- As per golint, we've redefined `func ServerBaseContext(h http.Handler, baseCtx context.Context) http.Handler` + to `func ServerBaseContext(baseCtx context.Context, h http.Handler) http.Handler` + + +## v3.1.0 (2017-07-10) + +- Fix a few minor issues after v3 release +- Move `docgen` sub-pkg to https://github.com/go-chi/docgen +- Move `render` sub-pkg to https://github.com/go-chi/render +- Add new `URLFormat` handler to chi/middleware sub-pkg to make working with url mime + suffixes easier, ie. parsing `/articles/1.json` and `/articles/1.xml`. See comments in + https://github.com/go-chi/chi/blob/master/middleware/url_format.go for example usage. + + +## v3.0.0 (2017-06-21) + +- Major update to chi library with many exciting updates, but also some *breaking changes* +- URL parameter syntax changed from `/:id` to `/{id}` for even more flexible routing, such as + `/articles/{month}-{day}-{year}-{slug}`, `/articles/{id}`, and `/articles/{id}.{ext}` on the + same router +- Support for regexp for routing patterns, in the form of `/{paramKey:regExp}` for example: + `r.Get("/articles/{name:[a-z]+}", h)` and `chi.URLParam(r, "name")` +- Add `Method` and `MethodFunc` to `chi.Router` to allow routing definitions such as + `r.Method("GET", "/", h)` which provides a cleaner interface for custom handlers like + in `_examples/custom-handler` +- Deprecating `mux#FileServer` helper function. Instead, we encourage users to create their + own using file handler with the stdlib, see `_examples/fileserver` for an example +- Add support for LINK/UNLINK http methods via `r.Method()` and `r.MethodFunc()` +- Moved the chi project to its own organization, to allow chi-related community packages to + be easily discovered and supported, at: https://github.com/go-chi +- *NOTE:* please update your import paths to `"github.com/go-chi/chi"` +- *NOTE:* chi v2 is still available at https://github.com/go-chi/chi/tree/v2 + + +## v2.1.0 (2017-03-30) + +- Minor improvements and update to the chi core library +- Introduced a brand new `chi/render` sub-package to complete the story of building + APIs to offer a pattern for managing well-defined request / response payloads. Please + check out the updated `_examples/rest` example for how it works. +- Added `MethodNotAllowed(h http.HandlerFunc)` to chi.Router interface + + +## v2.0.0 (2017-01-06) + +- After many months of v2 being in an RC state with many companies and users running it in + production, the inclusion of some improvements to the middlewares, we are very pleased to + announce v2.0.0 of chi. + + +## v2.0.0-rc1 (2016-07-26) + +- Huge update! chi v2 is a large refactor targetting Go 1.7+. As of Go 1.7, the popular + community `"net/context"` package has been included in the standard library as `"context"` and + utilized by `"net/http"` and `http.Request` to managing deadlines, cancelation signals and other + request-scoped values. We're very excited about the new context addition and are proud to + introduce chi v2, a minimal and powerful routing package for building large HTTP services, + with zero external dependencies. Chi focuses on idiomatic design and encourages the use of + stdlib HTTP handlers and middlwares. +- chi v2 deprecates its `chi.Handler` interface and requires `http.Handler` or `http.HandlerFunc` +- chi v2 stores URL routing parameters and patterns in the standard request context: `r.Context()` +- chi v2 lower-level routing context is accessible by `chi.RouteContext(r.Context()) *chi.Context`, + which provides direct access to URL routing parameters, the routing path and the matching + routing patterns. +- Users upgrading from chi v1 to v2, need to: + 1. Update the old chi.Handler signature, `func(ctx context.Context, w http.ResponseWriter, r *http.Request)` to + the standard http.Handler: `func(w http.ResponseWriter, r *http.Request)` + 2. Use `chi.URLParam(r *http.Request, paramKey string) string` + or `URLParamFromCtx(ctx context.Context, paramKey string) string` to access a url parameter value + + +## v1.0.0 (2016-07-01) + +- Released chi v1 stable https://github.com/go-chi/chi/tree/v1.0.0 for Go 1.6 and older. + + +## v0.9.0 (2016-03-31) + +- Reuse context objects via sync.Pool for zero-allocation routing [#33](https://github.com/go-chi/chi/pull/33) +- BREAKING NOTE: due to subtle API changes, previously `chi.URLParams(ctx)["id"]` used to access url parameters + has changed to: `chi.URLParam(ctx, "id")` diff --git a/vendor/github.com/go-chi/chi/CONTRIBUTING.md b/vendor/github.com/go-chi/chi/CONTRIBUTING.md new file mode 100644 index 00000000..c0ac2dfe --- /dev/null +++ b/vendor/github.com/go-chi/chi/CONTRIBUTING.md @@ -0,0 +1,31 @@ +# Contributing + +## Prerequisites + +1. [Install Go][go-install]. +2. Download the sources and switch the working directory: + + ```bash + go get -u -d github.com/go-chi/chi + cd $GOPATH/src/github.com/go-chi/chi + ``` + +## Submitting a Pull Request + +A typical workflow is: + +1. [Fork the repository.][fork] [This tip maybe also helpful.][go-fork-tip] +2. [Create a topic branch.][branch] +3. Add tests for your change. +4. Run `go test`. If your tests pass, return to the step 3. +5. Implement the change and ensure the steps from the previous step pass. +6. Run `goimports -w .`, to ensure the new code conforms to Go formatting guideline. +7. [Add, commit and push your changes.][git-help] +8. [Submit a pull request.][pull-req] + +[go-install]: https://golang.org/doc/install +[go-fork-tip]: http://blog.campoy.cat/2014/03/github-and-go-forking-pull-requests-and.html +[fork]: https://help.github.com/articles/fork-a-repo +[branch]: http://learn.github.com/p/branching.html +[git-help]: https://guides.github.com +[pull-req]: https://help.github.com/articles/using-pull-requests diff --git a/vendor/github.com/go-chi/chi/LICENSE b/vendor/github.com/go-chi/chi/LICENSE new file mode 100644 index 00000000..d99f02ff --- /dev/null +++ b/vendor/github.com/go-chi/chi/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2015-present Peter Kieltyka (https://github.com/pkieltyka), Google Inc. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/go-chi/chi/Makefile b/vendor/github.com/go-chi/chi/Makefile new file mode 100644 index 00000000..b96c92dd --- /dev/null +++ b/vendor/github.com/go-chi/chi/Makefile @@ -0,0 +1,14 @@ +all: + @echo "**********************************************************" + @echo "** chi build tool **" + @echo "**********************************************************" + + +test: + go clean -testcache && $(MAKE) test-router && $(MAKE) test-middleware + +test-router: + go test -race -v . + +test-middleware: + go test -race -v ./middleware diff --git a/vendor/github.com/go-chi/chi/README.md b/vendor/github.com/go-chi/chi/README.md new file mode 100644 index 00000000..1b96d360 --- /dev/null +++ b/vendor/github.com/go-chi/chi/README.md @@ -0,0 +1,511 @@ +# chi + + +[![GoDoc Widget]][GoDoc] [![Travis Widget]][Travis] + +`chi` is a lightweight, idiomatic and composable router for building Go HTTP services. It's +especially good at helping you write large REST API services that are kept maintainable as your +project grows and changes. `chi` is built on the new `context` package introduced in Go 1.7 to +handle signaling, cancelation and request-scoped values across a handler chain. + +The focus of the project has been to seek out an elegant and comfortable design for writing +REST API servers, written during the development of the Pressly API service that powers our +public API service, which in turn powers all of our client-side applications. + +The key considerations of chi's design are: project structure, maintainability, standard http +handlers (stdlib-only), developer productivity, and deconstructing a large system into many small +parts. The core router `github.com/go-chi/chi` is quite small (less than 1000 LOC), but we've also +included some useful/optional subpackages: [middleware](/middleware), [render](https://github.com/go-chi/render) +and [docgen](https://github.com/go-chi/docgen). We hope you enjoy it too! + +## Install + +`go get -u github.com/go-chi/chi` + + +## Features + +* **Lightweight** - cloc'd in ~1000 LOC for the chi router +* **Fast** - yes, see [benchmarks](#benchmarks) +* **100% compatible with net/http** - use any http or middleware pkg in the ecosystem that is also compatible with `net/http` +* **Designed for modular/composable APIs** - middlewares, inline middlewares, route groups and sub-router mounting +* **Context control** - built on new `context` package, providing value chaining, cancellations and timeouts +* **Robust** - in production at Pressly, CloudFlare, Heroku, 99Designs, and many others (see [discussion](https://github.com/go-chi/chi/issues/91)) +* **Doc generation** - `docgen` auto-generates routing documentation from your source to JSON or Markdown +* **Go.mod support** - v1.x of chi (starting from v1.5.0), now has go.mod support (see [CHANGELOG](https://github.com/go-chi/chi/blob/master/CHANGELOG.md#v150-2020-11-12---now-with-gomod-support)) +* **No external dependencies** - plain ol' Go stdlib + net/http + + +## Examples + +See [_examples/](https://github.com/go-chi/chi/blob/master/_examples/) for a variety of examples. + + +**As easy as:** + +```go +package main + +import ( + "net/http" + + "github.com/go-chi/chi" + "github.com/go-chi/chi/middleware" +) + +func main() { + r := chi.NewRouter() + r.Use(middleware.Logger) + r.Get("/", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("welcome")) + }) + http.ListenAndServe(":3000", r) +} +``` + +**REST Preview:** + +Here is a little preview of how routing looks like with chi. Also take a look at the generated routing docs +in JSON ([routes.json](https://github.com/go-chi/chi/blob/master/_examples/rest/routes.json)) and in +Markdown ([routes.md](https://github.com/go-chi/chi/blob/master/_examples/rest/routes.md)). + +I highly recommend reading the source of the [examples](https://github.com/go-chi/chi/blob/master/_examples/) listed +above, they will show you all the features of chi and serve as a good form of documentation. + +```go +import ( + //... + "context" + "github.com/go-chi/chi" + "github.com/go-chi/chi/middleware" +) + +func main() { + r := chi.NewRouter() + + // A good base middleware stack + r.Use(middleware.RequestID) + r.Use(middleware.RealIP) + r.Use(middleware.Logger) + r.Use(middleware.Recoverer) + + // Set a timeout value on the request context (ctx), that will signal + // through ctx.Done() that the request has timed out and further + // processing should be stopped. + r.Use(middleware.Timeout(60 * time.Second)) + + r.Get("/", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("hi")) + }) + + // RESTy routes for "articles" resource + r.Route("/articles", func(r chi.Router) { + r.With(paginate).Get("/", listArticles) // GET /articles + r.With(paginate).Get("/{month}-{day}-{year}", listArticlesByDate) // GET /articles/01-16-2017 + + r.Post("/", createArticle) // POST /articles + r.Get("/search", searchArticles) // GET /articles/search + + // Regexp url parameters: + r.Get("/{articleSlug:[a-z-]+}", getArticleBySlug) // GET /articles/home-is-toronto + + // Subrouters: + r.Route("/{articleID}", func(r chi.Router) { + r.Use(ArticleCtx) + r.Get("/", getArticle) // GET /articles/123 + r.Put("/", updateArticle) // PUT /articles/123 + r.Delete("/", deleteArticle) // DELETE /articles/123 + }) + }) + + // Mount the admin sub-router + r.Mount("/admin", adminRouter()) + + http.ListenAndServe(":3333", r) +} + +func ArticleCtx(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + articleID := chi.URLParam(r, "articleID") + article, err := dbGetArticle(articleID) + if err != nil { + http.Error(w, http.StatusText(404), 404) + return + } + ctx := context.WithValue(r.Context(), "article", article) + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +func getArticle(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + article, ok := ctx.Value("article").(*Article) + if !ok { + http.Error(w, http.StatusText(422), 422) + return + } + w.Write([]byte(fmt.Sprintf("title:%s", article.Title))) +} + +// A completely separate router for administrator routes +func adminRouter() http.Handler { + r := chi.NewRouter() + r.Use(AdminOnly) + r.Get("/", adminIndex) + r.Get("/accounts", adminListAccounts) + return r +} + +func AdminOnly(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + perm, ok := ctx.Value("acl.permission").(YourPermissionType) + if !ok || !perm.IsAdmin() { + http.Error(w, http.StatusText(403), 403) + return + } + next.ServeHTTP(w, r) + }) +} +``` + + +## Router interface + +chi's router is based on a kind of [Patricia Radix trie](https://en.wikipedia.org/wiki/Radix_tree). +The router is fully compatible with `net/http`. + +Built on top of the tree is the `Router` interface: + +```go +// Router consisting of the core routing methods used by chi's Mux, +// using only the standard net/http. +type Router interface { + http.Handler + Routes + + // Use appends one or more middlewares onto the Router stack. + Use(middlewares ...func(http.Handler) http.Handler) + + // With adds inline middlewares for an endpoint handler. + With(middlewares ...func(http.Handler) http.Handler) Router + + // Group adds a new inline-Router along the current routing + // path, with a fresh middleware stack for the inline-Router. + Group(fn func(r Router)) Router + + // Route mounts a sub-Router along a `pattern`` string. + Route(pattern string, fn func(r Router)) Router + + // Mount attaches another http.Handler along ./pattern/* + Mount(pattern string, h http.Handler) + + // Handle and HandleFunc adds routes for `pattern` that matches + // all HTTP methods. + Handle(pattern string, h http.Handler) + HandleFunc(pattern string, h http.HandlerFunc) + + // Method and MethodFunc adds routes for `pattern` that matches + // the `method` HTTP method. + Method(method, pattern string, h http.Handler) + MethodFunc(method, pattern string, h http.HandlerFunc) + + // HTTP-method routing along `pattern` + Connect(pattern string, h http.HandlerFunc) + Delete(pattern string, h http.HandlerFunc) + Get(pattern string, h http.HandlerFunc) + Head(pattern string, h http.HandlerFunc) + Options(pattern string, h http.HandlerFunc) + Patch(pattern string, h http.HandlerFunc) + Post(pattern string, h http.HandlerFunc) + Put(pattern string, h http.HandlerFunc) + Trace(pattern string, h http.HandlerFunc) + + // NotFound defines a handler to respond whenever a route could + // not be found. + NotFound(h http.HandlerFunc) + + // MethodNotAllowed defines a handler to respond whenever a method is + // not allowed. + MethodNotAllowed(h http.HandlerFunc) +} + +// Routes interface adds two methods for router traversal, which is also +// used by the github.com/go-chi/docgen package to generate documentation for Routers. +type Routes interface { + // Routes returns the routing tree in an easily traversable structure. + Routes() []Route + + // Middlewares returns the list of middlewares in use by the router. + Middlewares() Middlewares + + // Match searches the routing tree for a handler that matches + // the method/path - similar to routing a http request, but without + // executing the handler thereafter. + Match(rctx *Context, method, path string) bool +} +``` + +Each routing method accepts a URL `pattern` and chain of `handlers`. The URL pattern +supports named params (ie. `/users/{userID}`) and wildcards (ie. `/admin/*`). URL parameters +can be fetched at runtime by calling `chi.URLParam(r, "userID")` for named parameters +and `chi.URLParam(r, "*")` for a wildcard parameter. + + +### Middleware handlers + +chi's middlewares are just stdlib net/http middleware handlers. There is nothing special +about them, which means the router and all the tooling is designed to be compatible and +friendly with any middleware in the community. This offers much better extensibility and reuse +of packages and is at the heart of chi's purpose. + +Here is an example of a standard net/http middleware where we assign a context key `"user"` +the value of `"123"`. This middleware sets a hypothetical user identifier on the request +context and calls the next handler in the chain. + +```go +// HTTP middleware setting a value on the request context +func MyMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // create new context from `r` request context, and assign key `"user"` + // to value of `"123"` + ctx := context.WithValue(r.Context(), "user", "123") + + // call the next handler in the chain, passing the response writer and + // the updated request object with the new context value. + // + // note: context.Context values are nested, so any previously set + // values will be accessible as well, and the new `"user"` key + // will be accessible from this point forward. + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} +``` + + +### Request handlers + +chi uses standard net/http request handlers. This little snippet is an example of a http.Handler +func that reads a user identifier from the request context - hypothetically, identifying +the user sending an authenticated request, validated+set by a previous middleware handler. + +```go +// HTTP handler accessing data from the request context. +func MyRequestHandler(w http.ResponseWriter, r *http.Request) { + // here we read from the request context and fetch out `"user"` key set in + // the MyMiddleware example above. + user := r.Context().Value("user").(string) + + // respond to the client + w.Write([]byte(fmt.Sprintf("hi %s", user))) +} +``` + + +### URL parameters + +chi's router parses and stores URL parameters right onto the request context. Here is +an example of how to access URL params in your net/http handlers. And of course, middlewares +are able to access the same information. + +```go +// HTTP handler accessing the url routing parameters. +func MyRequestHandler(w http.ResponseWriter, r *http.Request) { + // fetch the url parameter `"userID"` from the request of a matching + // routing pattern. An example routing pattern could be: /users/{userID} + userID := chi.URLParam(r, "userID") + + // fetch `"key"` from the request context + ctx := r.Context() + key := ctx.Value("key").(string) + + // respond to the client + w.Write([]byte(fmt.Sprintf("hi %v, %v", userID, key))) +} +``` + + +## Middlewares + +chi comes equipped with an optional `middleware` package, providing a suite of standard +`net/http` middlewares. Please note, any middleware in the ecosystem that is also compatible +with `net/http` can be used with chi's mux. + +### Core middlewares + +---------------------------------------------------------------------------------------------------- +| chi/middleware Handler | description | +| :--------------------- | :---------------------------------------------------------------------- | +| [AllowContentEncoding] | Enforces a whitelist of request Content-Encoding headers | +| [AllowContentType] | Explicit whitelist of accepted request Content-Types | +| [BasicAuth] | Basic HTTP authentication | +| [Compress] | Gzip compression for clients that accept compressed responses | +| [ContentCharset] | Ensure charset for Content-Type request headers | +| [CleanPath] | Clean double slashes from request path | +| [GetHead] | Automatically route undefined HEAD requests to GET handlers | +| [Heartbeat] | Monitoring endpoint to check the servers pulse | +| [Logger] | Logs the start and end of each request with the elapsed processing time | +| [NoCache] | Sets response headers to prevent clients from caching | +| [Profiler] | Easily attach net/http/pprof to your routers | +| [RealIP] | Sets a http.Request's RemoteAddr to either X-Forwarded-For or X-Real-IP | +| [Recoverer] | Gracefully absorb panics and prints the stack trace | +| [RequestID] | Injects a request ID into the context of each request | +| [RedirectSlashes] | Redirect slashes on routing paths | +| [RouteHeaders] | Route handling for request headers | +| [SetHeader] | Short-hand middleware to set a response header key/value | +| [StripSlashes] | Strip slashes on routing paths | +| [Throttle] | Puts a ceiling on the number of concurrent requests | +| [Timeout] | Signals to the request context when the timeout deadline is reached | +| [URLFormat] | Parse extension from url and put it on request context | +| [WithValue] | Short-hand middleware to set a key/value on the request context | +---------------------------------------------------------------------------------------------------- + +[AllowContentEncoding]: https://pkg.go.dev/github.com/go-chi/chi/middleware#AllowContentEncoding +[AllowContentType]: https://pkg.go.dev/github.com/go-chi/chi/middleware#AllowContentType +[BasicAuth]: https://pkg.go.dev/github.com/go-chi/chi/middleware#BasicAuth +[Compress]: https://pkg.go.dev/github.com/go-chi/chi/middleware#Compress +[ContentCharset]: https://pkg.go.dev/github.com/go-chi/chi/middleware#ContentCharset +[CleanPath]: https://pkg.go.dev/github.com/go-chi/chi/middleware#CleanPath +[GetHead]: https://pkg.go.dev/github.com/go-chi/chi/middleware#GetHead +[GetReqID]: https://pkg.go.dev/github.com/go-chi/chi/middleware#GetReqID +[Heartbeat]: https://pkg.go.dev/github.com/go-chi/chi/middleware#Heartbeat +[Logger]: https://pkg.go.dev/github.com/go-chi/chi/middleware#Logger +[NoCache]: https://pkg.go.dev/github.com/go-chi/chi/middleware#NoCache +[Profiler]: https://pkg.go.dev/github.com/go-chi/chi/middleware#Profiler +[RealIP]: https://pkg.go.dev/github.com/go-chi/chi/middleware#RealIP +[Recoverer]: https://pkg.go.dev/github.com/go-chi/chi/middleware#Recoverer +[RedirectSlashes]: https://pkg.go.dev/github.com/go-chi/chi/middleware#RedirectSlashes +[RequestLogger]: https://pkg.go.dev/github.com/go-chi/chi/middleware#RequestLogger +[RequestID]: https://pkg.go.dev/github.com/go-chi/chi/middleware#RequestID +[RouteHeaders]: https://pkg.go.dev/github.com/go-chi/chi/middleware#RouteHeaders +[SetHeader]: https://pkg.go.dev/github.com/go-chi/chi/middleware#SetHeader +[StripSlashes]: https://pkg.go.dev/github.com/go-chi/chi/middleware#StripSlashes +[Throttle]: https://pkg.go.dev/github.com/go-chi/chi/middleware#Throttle +[ThrottleBacklog]: https://pkg.go.dev/github.com/go-chi/chi/middleware#ThrottleBacklog +[ThrottleWithOpts]: https://pkg.go.dev/github.com/go-chi/chi/middleware#ThrottleWithOpts +[Timeout]: https://pkg.go.dev/github.com/go-chi/chi/middleware#Timeout +[URLFormat]: https://pkg.go.dev/github.com/go-chi/chi/middleware#URLFormat +[WithLogEntry]: https://pkg.go.dev/github.com/go-chi/chi/middleware#WithLogEntry +[WithValue]: https://pkg.go.dev/github.com/go-chi/chi/middleware#WithValue +[Compressor]: https://pkg.go.dev/github.com/go-chi/chi/middleware#Compressor +[DefaultLogFormatter]: https://pkg.go.dev/github.com/go-chi/chi/middleware#DefaultLogFormatter +[EncoderFunc]: https://pkg.go.dev/github.com/go-chi/chi/middleware#EncoderFunc +[HeaderRoute]: https://pkg.go.dev/github.com/go-chi/chi/middleware#HeaderRoute +[HeaderRouter]: https://pkg.go.dev/github.com/go-chi/chi/middleware#HeaderRouter +[LogEntry]: https://pkg.go.dev/github.com/go-chi/chi/middleware#LogEntry +[LogFormatter]: https://pkg.go.dev/github.com/go-chi/chi/middleware#LogFormatter +[LoggerInterface]: https://pkg.go.dev/github.com/go-chi/chi/middleware#LoggerInterface +[ThrottleOpts]: https://pkg.go.dev/github.com/go-chi/chi/middleware#ThrottleOpts +[WrapResponseWriter]: https://pkg.go.dev/github.com/go-chi/chi/middleware#WrapResponseWriter + +### Extra middlewares & packages + +Please see https://github.com/go-chi for additional packages. + +-------------------------------------------------------------------------------------------------------------------- +| package | description | +|:---------------------------------------------------|:------------------------------------------------------------- +| [cors](https://github.com/go-chi/cors) | Cross-origin resource sharing (CORS) | +| [docgen](https://github.com/go-chi/docgen) | Print chi.Router routes at runtime | +| [jwtauth](https://github.com/go-chi/jwtauth) | JWT authentication | +| [hostrouter](https://github.com/go-chi/hostrouter) | Domain/host based request routing | +| [httplog](https://github.com/go-chi/httplog) | Small but powerful structured HTTP request logging | +| [httprate](https://github.com/go-chi/httprate) | HTTP request rate limiter | +| [httptracer](https://github.com/go-chi/httptracer) | HTTP request performance tracing library | +| [httpvcr](https://github.com/go-chi/httpvcr) | Write deterministic tests for external sources | +| [stampede](https://github.com/go-chi/stampede) | HTTP request coalescer | +-------------------------------------------------------------------------------------------------------------------- + + +## context? + +`context` is a tiny pkg that provides simple interface to signal context across call stacks +and goroutines. It was originally written by [Sameer Ajmani](https://github.com/Sajmani) +and is available in stdlib since go1.7. + +Learn more at https://blog.golang.org/context + +and.. +* Docs: https://golang.org/pkg/context +* Source: https://github.com/golang/go/tree/master/src/context + + +## Benchmarks + +The benchmark suite: https://github.com/pkieltyka/go-http-routing-benchmark + +Results as of Nov 29, 2020 with Go 1.15.5 on Linux AMD 3950x + +```shell +BenchmarkChi_Param 3075895 384 ns/op 400 B/op 2 allocs/op +BenchmarkChi_Param5 2116603 566 ns/op 400 B/op 2 allocs/op +BenchmarkChi_Param20 964117 1227 ns/op 400 B/op 2 allocs/op +BenchmarkChi_ParamWrite 2863413 420 ns/op 400 B/op 2 allocs/op +BenchmarkChi_GithubStatic 3045488 395 ns/op 400 B/op 2 allocs/op +BenchmarkChi_GithubParam 2204115 540 ns/op 400 B/op 2 allocs/op +BenchmarkChi_GithubAll 10000 113811 ns/op 81203 B/op 406 allocs/op +BenchmarkChi_GPlusStatic 3337485 359 ns/op 400 B/op 2 allocs/op +BenchmarkChi_GPlusParam 2825853 423 ns/op 400 B/op 2 allocs/op +BenchmarkChi_GPlus2Params 2471697 483 ns/op 400 B/op 2 allocs/op +BenchmarkChi_GPlusAll 194220 5950 ns/op 5200 B/op 26 allocs/op +BenchmarkChi_ParseStatic 3365324 356 ns/op 400 B/op 2 allocs/op +BenchmarkChi_ParseParam 2976614 404 ns/op 400 B/op 2 allocs/op +BenchmarkChi_Parse2Params 2638084 439 ns/op 400 B/op 2 allocs/op +BenchmarkChi_ParseAll 109567 11295 ns/op 10400 B/op 52 allocs/op +BenchmarkChi_StaticAll 16846 71308 ns/op 62802 B/op 314 allocs/op +``` + +Comparison with other routers: https://gist.github.com/pkieltyka/123032f12052520aaccab752bd3e78cc + +NOTE: the allocs in the benchmark above are from the calls to http.Request's +`WithContext(context.Context)` method that clones the http.Request, sets the `Context()` +on the duplicated (alloc'd) request and returns it the new request object. This is just +how setting context on a request in Go works. + + +## Go module support & note on chi's versioning + +* Go.mod support means we reset our versioning starting from v1.5 (see [CHANGELOG](https://github.com/go-chi/chi/blob/master/CHANGELOG.md#v150-2020-11-12---now-with-gomod-support)) +* All older tags are preserved, are backwards-compatible and will "just work" as they +* Brand new systems can run `go get -u github.com/go-chi/chi` as normal, or `go get -u github.com/go-chi/chi@latest` +to install chi, which will install v1.x+ built with go.mod support, starting from v1.5.0. +* For existing projects who want to upgrade to the latest go.mod version, run: `go get -u github.com/go-chi/chi@v1.5.0`, +which will get you on the go.mod version line (as Go's mod cache may still remember v4.x). +* Any breaking changes will bump a "minor" release and backwards-compatible improvements/fixes will bump a "tiny" release. + + +## Credits + +* Carl Jackson for https://github.com/zenazn/goji + * Parts of chi's thinking comes from goji, and chi's middleware package + sources from goji. +* Armon Dadgar for https://github.com/armon/go-radix +* Contributions: [@VojtechVitek](https://github.com/VojtechVitek) + +We'll be more than happy to see [your contributions](./CONTRIBUTING.md)! + + +## Beyond REST + +chi is just a http router that lets you decompose request handling into many smaller layers. +Many companies use chi to write REST services for their public APIs. But, REST is just a convention +for managing state via HTTP, and there's a lot of other pieces required to write a complete client-server +system or network of microservices. + +Looking beyond REST, I also recommend some newer works in the field: +* [webrpc](https://github.com/webrpc/webrpc) - Web-focused RPC client+server framework with code-gen +* [gRPC](https://github.com/grpc/grpc-go) - Google's RPC framework via protobufs +* [graphql](https://github.com/99designs/gqlgen) - Declarative query language +* [NATS](https://nats.io) - lightweight pub-sub + + +## License + +Copyright (c) 2015-present [Peter Kieltyka](https://github.com/pkieltyka) + +Licensed under [MIT License](./LICENSE) + +[GoDoc]: https://pkg.go.dev/github.com/go-chi/chi?tab=versions +[GoDoc Widget]: https://godoc.org/github.com/go-chi/chi?status.svg +[Travis]: https://travis-ci.org/go-chi/chi +[Travis Widget]: https://travis-ci.org/go-chi/chi.svg?branch=master diff --git a/vendor/github.com/go-chi/chi/chain.go b/vendor/github.com/go-chi/chi/chain.go new file mode 100644 index 00000000..88e68461 --- /dev/null +++ b/vendor/github.com/go-chi/chi/chain.go @@ -0,0 +1,49 @@ +package chi + +import "net/http" + +// Chain returns a Middlewares type from a slice of middleware handlers. +func Chain(middlewares ...func(http.Handler) http.Handler) Middlewares { + return Middlewares(middlewares) +} + +// Handler builds and returns a http.Handler from the chain of middlewares, +// with `h http.Handler` as the final handler. +func (mws Middlewares) Handler(h http.Handler) http.Handler { + return &ChainHandler{mws, h, chain(mws, h)} +} + +// HandlerFunc builds and returns a http.Handler from the chain of middlewares, +// with `h http.Handler` as the final handler. +func (mws Middlewares) HandlerFunc(h http.HandlerFunc) http.Handler { + return &ChainHandler{mws, h, chain(mws, h)} +} + +// ChainHandler is a http.Handler with support for handler composition and +// execution. +type ChainHandler struct { + Middlewares Middlewares + Endpoint http.Handler + chain http.Handler +} + +func (c *ChainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + c.chain.ServeHTTP(w, r) +} + +// chain builds a http.Handler composed of an inline middleware stack and endpoint +// handler in the order they are passed. +func chain(middlewares []func(http.Handler) http.Handler, endpoint http.Handler) http.Handler { + // Return ahead of time if there aren't any middlewares for the chain + if len(middlewares) == 0 { + return endpoint + } + + // Wrap the end handler with the middleware chain + h := middlewares[len(middlewares)-1](endpoint) + for i := len(middlewares) - 2; i >= 0; i-- { + h = middlewares[i](h) + } + + return h +} diff --git a/vendor/github.com/go-chi/chi/chi.go b/vendor/github.com/go-chi/chi/chi.go new file mode 100644 index 00000000..b7063dc2 --- /dev/null +++ b/vendor/github.com/go-chi/chi/chi.go @@ -0,0 +1,134 @@ +// +// Package chi is a small, idiomatic and composable router for building HTTP services. +// +// chi requires Go 1.10 or newer. +// +// Example: +// package main +// +// import ( +// "net/http" +// +// "github.com/go-chi/chi" +// "github.com/go-chi/chi/middleware" +// ) +// +// func main() { +// r := chi.NewRouter() +// r.Use(middleware.Logger) +// r.Use(middleware.Recoverer) +// +// r.Get("/", func(w http.ResponseWriter, r *http.Request) { +// w.Write([]byte("root.")) +// }) +// +// http.ListenAndServe(":3333", r) +// } +// +// See github.com/go-chi/chi/_examples/ for more in-depth examples. +// +// URL patterns allow for easy matching of path components in HTTP +// requests. The matching components can then be accessed using +// chi.URLParam(). All patterns must begin with a slash. +// +// A simple named placeholder {name} matches any sequence of characters +// up to the next / or the end of the URL. Trailing slashes on paths must +// be handled explicitly. +// +// A placeholder with a name followed by a colon allows a regular +// expression match, for example {number:\\d+}. The regular expression +// syntax is Go's normal regexp RE2 syntax, except that regular expressions +// including { or } are not supported, and / will never be +// matched. An anonymous regexp pattern is allowed, using an empty string +// before the colon in the placeholder, such as {:\\d+} +// +// The special placeholder of asterisk matches the rest of the requested +// URL. Any trailing characters in the pattern are ignored. This is the only +// placeholder which will match / characters. +// +// Examples: +// "/user/{name}" matches "/user/jsmith" but not "/user/jsmith/info" or "/user/jsmith/" +// "/user/{name}/info" matches "/user/jsmith/info" +// "/page/*" matches "/page/intro/latest" +// "/page/*/index" also matches "/page/intro/latest" +// "/date/{yyyy:\\d\\d\\d\\d}/{mm:\\d\\d}/{dd:\\d\\d}" matches "/date/2017/04/01" +// +package chi + +import "net/http" + +// NewRouter returns a new Mux object that implements the Router interface. +func NewRouter() *Mux { + return NewMux() +} + +// Router consisting of the core routing methods used by chi's Mux, +// using only the standard net/http. +type Router interface { + http.Handler + Routes + + // Use appends one or more middlewares onto the Router stack. + Use(middlewares ...func(http.Handler) http.Handler) + + // With adds inline middlewares for an endpoint handler. + With(middlewares ...func(http.Handler) http.Handler) Router + + // Group adds a new inline-Router along the current routing + // path, with a fresh middleware stack for the inline-Router. + Group(fn func(r Router)) Router + + // Route mounts a sub-Router along a `pattern`` string. + Route(pattern string, fn func(r Router)) Router + + // Mount attaches another http.Handler along ./pattern/* + Mount(pattern string, h http.Handler) + + // Handle and HandleFunc adds routes for `pattern` that matches + // all HTTP methods. + Handle(pattern string, h http.Handler) + HandleFunc(pattern string, h http.HandlerFunc) + + // Method and MethodFunc adds routes for `pattern` that matches + // the `method` HTTP method. + Method(method, pattern string, h http.Handler) + MethodFunc(method, pattern string, h http.HandlerFunc) + + // HTTP-method routing along `pattern` + Connect(pattern string, h http.HandlerFunc) + Delete(pattern string, h http.HandlerFunc) + Get(pattern string, h http.HandlerFunc) + Head(pattern string, h http.HandlerFunc) + Options(pattern string, h http.HandlerFunc) + Patch(pattern string, h http.HandlerFunc) + Post(pattern string, h http.HandlerFunc) + Put(pattern string, h http.HandlerFunc) + Trace(pattern string, h http.HandlerFunc) + + // NotFound defines a handler to respond whenever a route could + // not be found. + NotFound(h http.HandlerFunc) + + // MethodNotAllowed defines a handler to respond whenever a method is + // not allowed. + MethodNotAllowed(h http.HandlerFunc) +} + +// Routes interface adds two methods for router traversal, which is also +// used by the `docgen` subpackage to generation documentation for Routers. +type Routes interface { + // Routes returns the routing tree in an easily traversable structure. + Routes() []Route + + // Middlewares returns the list of middlewares in use by the router. + Middlewares() Middlewares + + // Match searches the routing tree for a handler that matches + // the method/path - similar to routing a http request, but without + // executing the handler thereafter. + Match(rctx *Context, method, path string) bool +} + +// Middlewares type is a slice of standard middleware handlers with methods +// to compose middleware chains and http.Handler's. +type Middlewares []func(http.Handler) http.Handler diff --git a/vendor/github.com/go-chi/chi/context.go b/vendor/github.com/go-chi/chi/context.go new file mode 100644 index 00000000..8c97f214 --- /dev/null +++ b/vendor/github.com/go-chi/chi/context.go @@ -0,0 +1,157 @@ +package chi + +import ( + "context" + "net/http" + "strings" +) + +// URLParam returns the url parameter from a http.Request object. +func URLParam(r *http.Request, key string) string { + if rctx := RouteContext(r.Context()); rctx != nil { + return rctx.URLParam(key) + } + return "" +} + +// URLParamFromCtx returns the url parameter from a http.Request Context. +func URLParamFromCtx(ctx context.Context, key string) string { + if rctx := RouteContext(ctx); rctx != nil { + return rctx.URLParam(key) + } + return "" +} + +// RouteContext returns chi's routing Context object from a +// http.Request Context. +func RouteContext(ctx context.Context) *Context { + val, _ := ctx.Value(RouteCtxKey).(*Context) + return val +} + +// NewRouteContext returns a new routing Context object. +func NewRouteContext() *Context { + return &Context{} +} + +var ( + // RouteCtxKey is the context.Context key to store the request context. + RouteCtxKey = &contextKey{"RouteContext"} +) + +// Context is the default routing context set on the root node of a +// request context to track route patterns, URL parameters and +// an optional routing path. +type Context struct { + Routes Routes + + // Routing path/method override used during the route search. + // See Mux#routeHTTP method. + RoutePath string + RouteMethod string + + // Routing pattern stack throughout the lifecycle of the request, + // across all connected routers. It is a record of all matching + // patterns across a stack of sub-routers. + RoutePatterns []string + + // URLParams are the stack of routeParams captured during the + // routing lifecycle across a stack of sub-routers. + URLParams RouteParams + + // The endpoint routing pattern that matched the request URI path + // or `RoutePath` of the current sub-router. This value will update + // during the lifecycle of a request passing through a stack of + // sub-routers. + routePattern string + + // Route parameters matched for the current sub-router. It is + // intentionally unexported so it cant be tampered. + routeParams RouteParams + + // methodNotAllowed hint + methodNotAllowed bool + + // parentCtx is the parent of this one, for using Context as a + // context.Context directly. This is an optimization that saves + // 1 allocation. + parentCtx context.Context +} + +// Reset a routing context to its initial state. +func (x *Context) Reset() { + x.Routes = nil + x.RoutePath = "" + x.RouteMethod = "" + x.RoutePatterns = x.RoutePatterns[:0] + x.URLParams.Keys = x.URLParams.Keys[:0] + x.URLParams.Values = x.URLParams.Values[:0] + + x.routePattern = "" + x.routeParams.Keys = x.routeParams.Keys[:0] + x.routeParams.Values = x.routeParams.Values[:0] + x.methodNotAllowed = false + x.parentCtx = nil +} + +// URLParam returns the corresponding URL parameter value from the request +// routing context. +func (x *Context) URLParam(key string) string { + for k := len(x.URLParams.Keys) - 1; k >= 0; k-- { + if x.URLParams.Keys[k] == key { + return x.URLParams.Values[k] + } + } + return "" +} + +// RoutePattern builds the routing pattern string for the particular +// request, at the particular point during routing. This means, the value +// will change throughout the execution of a request in a router. That is +// why its advised to only use this value after calling the next handler. +// +// For example, +// +// func Instrument(next http.Handler) http.Handler { +// return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +// next.ServeHTTP(w, r) +// routePattern := chi.RouteContext(r.Context()).RoutePattern() +// measure(w, r, routePattern) +// }) +// } +func (x *Context) RoutePattern() string { + routePattern := strings.Join(x.RoutePatterns, "") + return replaceWildcards(routePattern) +} + +// replaceWildcards takes a route pattern and recursively replaces all +// occurrences of "/*/" to "/". +func replaceWildcards(p string) string { + if strings.Contains(p, "/*/") { + return replaceWildcards(strings.Replace(p, "/*/", "/", -1)) + } + + return p +} + +// RouteParams is a structure to track URL routing parameters efficiently. +type RouteParams struct { + Keys, Values []string +} + +// Add will append a URL parameter to the end of the route param +func (s *RouteParams) Add(key, value string) { + s.Keys = append(s.Keys, key) + s.Values = append(s.Values, value) +} + +// contextKey is a value for use with context.WithValue. It's used as +// a pointer so it fits in an interface{} without allocation. This technique +// for defining context keys was copied from Go 1.7's new use of context in net/http. +type contextKey struct { + name string +} + +func (k *contextKey) String() string { + return "chi context value " + k.name +} diff --git a/vendor/github.com/go-chi/chi/mux.go b/vendor/github.com/go-chi/chi/mux.go new file mode 100644 index 00000000..146643b0 --- /dev/null +++ b/vendor/github.com/go-chi/chi/mux.go @@ -0,0 +1,479 @@ +package chi + +import ( + "context" + "fmt" + "net/http" + "strings" + "sync" +) + +var _ Router = &Mux{} + +// Mux is a simple HTTP route multiplexer that parses a request path, +// records any URL params, and executes an end handler. It implements +// the http.Handler interface and is friendly with the standard library. +// +// Mux is designed to be fast, minimal and offer a powerful API for building +// modular and composable HTTP services with a large set of handlers. It's +// particularly useful for writing large REST API services that break a handler +// into many smaller parts composed of middlewares and end handlers. +type Mux struct { + // The radix trie router + tree *node + + // The middleware stack + middlewares []func(http.Handler) http.Handler + + // Controls the behaviour of middleware chain generation when a mux + // is registered as an inline group inside another mux. + inline bool + parent *Mux + + // The computed mux handler made of the chained middleware stack and + // the tree router + handler http.Handler + + // Routing context pool + pool *sync.Pool + + // Custom route not found handler + notFoundHandler http.HandlerFunc + + // Custom method not allowed handler + methodNotAllowedHandler http.HandlerFunc +} + +// NewMux returns a newly initialized Mux object that implements the Router +// interface. +func NewMux() *Mux { + mux := &Mux{tree: &node{}, pool: &sync.Pool{}} + mux.pool.New = func() interface{} { + return NewRouteContext() + } + return mux +} + +// ServeHTTP is the single method of the http.Handler interface that makes +// Mux interoperable with the standard library. It uses a sync.Pool to get and +// reuse routing contexts for each request. +func (mx *Mux) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Ensure the mux has some routes defined on the mux + if mx.handler == nil { + mx.NotFoundHandler().ServeHTTP(w, r) + return + } + + // Check if a routing context already exists from a parent router. + rctx, _ := r.Context().Value(RouteCtxKey).(*Context) + if rctx != nil { + mx.handler.ServeHTTP(w, r) + return + } + + // Fetch a RouteContext object from the sync pool, and call the computed + // mx.handler that is comprised of mx.middlewares + mx.routeHTTP. + // Once the request is finished, reset the routing context and put it back + // into the pool for reuse from another request. + rctx = mx.pool.Get().(*Context) + rctx.Reset() + rctx.Routes = mx + rctx.parentCtx = r.Context() + + // NOTE: r.WithContext() causes 2 allocations and context.WithValue() causes 1 allocation + r = r.WithContext(context.WithValue(r.Context(), RouteCtxKey, rctx)) + + // Serve the request and once its done, put the request context back in the sync pool + mx.handler.ServeHTTP(w, r) + mx.pool.Put(rctx) +} + +// Use appends a middleware handler to the Mux middleware stack. +// +// The middleware stack for any Mux will execute before searching for a matching +// route to a specific handler, which provides opportunity to respond early, +// change the course of the request execution, or set request-scoped values for +// the next http.Handler. +func (mx *Mux) Use(middlewares ...func(http.Handler) http.Handler) { + if mx.handler != nil { + panic("chi: all middlewares must be defined before routes on a mux") + } + mx.middlewares = append(mx.middlewares, middlewares...) +} + +// Handle adds the route `pattern` that matches any http method to +// execute the `handler` http.Handler. +func (mx *Mux) Handle(pattern string, handler http.Handler) { + mx.handle(mALL, pattern, handler) +} + +// HandleFunc adds the route `pattern` that matches any http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) HandleFunc(pattern string, handlerFn http.HandlerFunc) { + mx.handle(mALL, pattern, handlerFn) +} + +// Method adds the route `pattern` that matches `method` http method to +// execute the `handler` http.Handler. +func (mx *Mux) Method(method, pattern string, handler http.Handler) { + m, ok := methodMap[strings.ToUpper(method)] + if !ok { + panic(fmt.Sprintf("chi: '%s' http method is not supported.", method)) + } + mx.handle(m, pattern, handler) +} + +// MethodFunc adds the route `pattern` that matches `method` http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) MethodFunc(method, pattern string, handlerFn http.HandlerFunc) { + mx.Method(method, pattern, handlerFn) +} + +// Connect adds the route `pattern` that matches a CONNECT http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) Connect(pattern string, handlerFn http.HandlerFunc) { + mx.handle(mCONNECT, pattern, handlerFn) +} + +// Delete adds the route `pattern` that matches a DELETE http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) Delete(pattern string, handlerFn http.HandlerFunc) { + mx.handle(mDELETE, pattern, handlerFn) +} + +// Get adds the route `pattern` that matches a GET http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) Get(pattern string, handlerFn http.HandlerFunc) { + mx.handle(mGET, pattern, handlerFn) +} + +// Head adds the route `pattern` that matches a HEAD http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) Head(pattern string, handlerFn http.HandlerFunc) { + mx.handle(mHEAD, pattern, handlerFn) +} + +// Options adds the route `pattern` that matches a OPTIONS http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) Options(pattern string, handlerFn http.HandlerFunc) { + mx.handle(mOPTIONS, pattern, handlerFn) +} + +// Patch adds the route `pattern` that matches a PATCH http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) Patch(pattern string, handlerFn http.HandlerFunc) { + mx.handle(mPATCH, pattern, handlerFn) +} + +// Post adds the route `pattern` that matches a POST http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) Post(pattern string, handlerFn http.HandlerFunc) { + mx.handle(mPOST, pattern, handlerFn) +} + +// Put adds the route `pattern` that matches a PUT http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) Put(pattern string, handlerFn http.HandlerFunc) { + mx.handle(mPUT, pattern, handlerFn) +} + +// Trace adds the route `pattern` that matches a TRACE http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) Trace(pattern string, handlerFn http.HandlerFunc) { + mx.handle(mTRACE, pattern, handlerFn) +} + +// NotFound sets a custom http.HandlerFunc for routing paths that could +// not be found. The default 404 handler is `http.NotFound`. +func (mx *Mux) NotFound(handlerFn http.HandlerFunc) { + // Build NotFound handler chain + m := mx + h := Chain(mx.middlewares...).HandlerFunc(handlerFn).ServeHTTP + if mx.inline && mx.parent != nil { + m = mx.parent + } + + // Update the notFoundHandler from this point forward + m.notFoundHandler = h + m.updateSubRoutes(func(subMux *Mux) { + if subMux.notFoundHandler == nil { + subMux.NotFound(h) + } + }) +} + +// MethodNotAllowed sets a custom http.HandlerFunc for routing paths where the +// method is unresolved. The default handler returns a 405 with an empty body. +func (mx *Mux) MethodNotAllowed(handlerFn http.HandlerFunc) { + // Build MethodNotAllowed handler chain + m := mx + h := Chain(mx.middlewares...).HandlerFunc(handlerFn).ServeHTTP + if mx.inline && mx.parent != nil { + m = mx.parent + } + + // Update the methodNotAllowedHandler from this point forward + m.methodNotAllowedHandler = h + m.updateSubRoutes(func(subMux *Mux) { + if subMux.methodNotAllowedHandler == nil { + subMux.MethodNotAllowed(h) + } + }) +} + +// With adds inline middlewares for an endpoint handler. +func (mx *Mux) With(middlewares ...func(http.Handler) http.Handler) Router { + // Similarly as in handle(), we must build the mux handler once additional + // middleware registration isn't allowed for this stack, like now. + if !mx.inline && mx.handler == nil { + mx.updateRouteHandler() + } + + // Copy middlewares from parent inline muxs + var mws Middlewares + if mx.inline { + mws = make(Middlewares, len(mx.middlewares)) + copy(mws, mx.middlewares) + } + mws = append(mws, middlewares...) + + im := &Mux{ + pool: mx.pool, inline: true, parent: mx, tree: mx.tree, middlewares: mws, + notFoundHandler: mx.notFoundHandler, methodNotAllowedHandler: mx.methodNotAllowedHandler, + } + + return im +} + +// Group creates a new inline-Mux with a fresh middleware stack. It's useful +// for a group of handlers along the same routing path that use an additional +// set of middlewares. See _examples/. +func (mx *Mux) Group(fn func(r Router)) Router { + im := mx.With().(*Mux) + if fn != nil { + fn(im) + } + return im +} + +// Route creates a new Mux with a fresh middleware stack and mounts it +// along the `pattern` as a subrouter. Effectively, this is a short-hand +// call to Mount. See _examples/. +func (mx *Mux) Route(pattern string, fn func(r Router)) Router { + if fn == nil { + panic(fmt.Sprintf("chi: attempting to Route() a nil subrouter on '%s'", pattern)) + } + subRouter := NewRouter() + fn(subRouter) + mx.Mount(pattern, subRouter) + return subRouter +} + +// Mount attaches another http.Handler or chi Router as a subrouter along a routing +// path. It's very useful to split up a large API as many independent routers and +// compose them as a single service using Mount. See _examples/. +// +// Note that Mount() simply sets a wildcard along the `pattern` that will continue +// routing at the `handler`, which in most cases is another chi.Router. As a result, +// if you define two Mount() routes on the exact same pattern the mount will panic. +func (mx *Mux) Mount(pattern string, handler http.Handler) { + if handler == nil { + panic(fmt.Sprintf("chi: attempting to Mount() a nil handler on '%s'", pattern)) + } + + // Provide runtime safety for ensuring a pattern isn't mounted on an existing + // routing pattern. + if mx.tree.findPattern(pattern+"*") || mx.tree.findPattern(pattern+"/*") { + panic(fmt.Sprintf("chi: attempting to Mount() a handler on an existing path, '%s'", pattern)) + } + + // Assign sub-Router's with the parent not found & method not allowed handler if not specified. + subr, ok := handler.(*Mux) + if ok && subr.notFoundHandler == nil && mx.notFoundHandler != nil { + subr.NotFound(mx.notFoundHandler) + } + if ok && subr.methodNotAllowedHandler == nil && mx.methodNotAllowedHandler != nil { + subr.MethodNotAllowed(mx.methodNotAllowedHandler) + } + + mountHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rctx := RouteContext(r.Context()) + + // shift the url path past the previous subrouter + rctx.RoutePath = mx.nextRoutePath(rctx) + + // reset the wildcard URLParam which connects the subrouter + n := len(rctx.URLParams.Keys) - 1 + if n >= 0 && rctx.URLParams.Keys[n] == "*" && len(rctx.URLParams.Values) > n { + rctx.URLParams.Values[n] = "" + } + + handler.ServeHTTP(w, r) + }) + + if pattern == "" || pattern[len(pattern)-1] != '/' { + mx.handle(mALL|mSTUB, pattern, mountHandler) + mx.handle(mALL|mSTUB, pattern+"/", mountHandler) + pattern += "/" + } + + method := mALL + subroutes, _ := handler.(Routes) + if subroutes != nil { + method |= mSTUB + } + n := mx.handle(method, pattern+"*", mountHandler) + + if subroutes != nil { + n.subroutes = subroutes + } +} + +// Routes returns a slice of routing information from the tree, +// useful for traversing available routes of a router. +func (mx *Mux) Routes() []Route { + return mx.tree.routes() +} + +// Middlewares returns a slice of middleware handler functions. +func (mx *Mux) Middlewares() Middlewares { + return mx.middlewares +} + +// Match searches the routing tree for a handler that matches the method/path. +// It's similar to routing a http request, but without executing the handler +// thereafter. +// +// Note: the *Context state is updated during execution, so manage +// the state carefully or make a NewRouteContext(). +func (mx *Mux) Match(rctx *Context, method, path string) bool { + m, ok := methodMap[method] + if !ok { + return false + } + + node, _, h := mx.tree.FindRoute(rctx, m, path) + + if node != nil && node.subroutes != nil { + rctx.RoutePath = mx.nextRoutePath(rctx) + return node.subroutes.Match(rctx, method, rctx.RoutePath) + } + + return h != nil +} + +// NotFoundHandler returns the default Mux 404 responder whenever a route +// cannot be found. +func (mx *Mux) NotFoundHandler() http.HandlerFunc { + if mx.notFoundHandler != nil { + return mx.notFoundHandler + } + return http.NotFound +} + +// MethodNotAllowedHandler returns the default Mux 405 responder whenever +// a method cannot be resolved for a route. +func (mx *Mux) MethodNotAllowedHandler() http.HandlerFunc { + if mx.methodNotAllowedHandler != nil { + return mx.methodNotAllowedHandler + } + return methodNotAllowedHandler +} + +// handle registers a http.Handler in the routing tree for a particular http method +// and routing pattern. +func (mx *Mux) handle(method methodTyp, pattern string, handler http.Handler) *node { + if len(pattern) == 0 || pattern[0] != '/' { + panic(fmt.Sprintf("chi: routing pattern must begin with '/' in '%s'", pattern)) + } + + // Build the computed routing handler for this routing pattern. + if !mx.inline && mx.handler == nil { + mx.updateRouteHandler() + } + + // Build endpoint handler with inline middlewares for the route + var h http.Handler + if mx.inline { + mx.handler = http.HandlerFunc(mx.routeHTTP) + h = Chain(mx.middlewares...).Handler(handler) + } else { + h = handler + } + + // Add the endpoint to the tree and return the node + return mx.tree.InsertRoute(method, pattern, h) +} + +// routeHTTP routes a http.Request through the Mux routing tree to serve +// the matching handler for a particular http method. +func (mx *Mux) routeHTTP(w http.ResponseWriter, r *http.Request) { + // Grab the route context object + rctx := r.Context().Value(RouteCtxKey).(*Context) + + // The request routing path + routePath := rctx.RoutePath + if routePath == "" { + if r.URL.RawPath != "" { + routePath = r.URL.RawPath + } else { + routePath = r.URL.Path + } + } + + // Check if method is supported by chi + if rctx.RouteMethod == "" { + rctx.RouteMethod = r.Method + } + method, ok := methodMap[rctx.RouteMethod] + if !ok { + mx.MethodNotAllowedHandler().ServeHTTP(w, r) + return + } + + // Find the route + if _, _, h := mx.tree.FindRoute(rctx, method, routePath); h != nil { + h.ServeHTTP(w, r) + return + } + if rctx.methodNotAllowed { + mx.MethodNotAllowedHandler().ServeHTTP(w, r) + } else { + mx.NotFoundHandler().ServeHTTP(w, r) + } +} + +func (mx *Mux) nextRoutePath(rctx *Context) string { + routePath := "/" + nx := len(rctx.routeParams.Keys) - 1 // index of last param in list + if nx >= 0 && rctx.routeParams.Keys[nx] == "*" && len(rctx.routeParams.Values) > nx { + routePath = "/" + rctx.routeParams.Values[nx] + } + return routePath +} + +// Recursively update data on child routers. +func (mx *Mux) updateSubRoutes(fn func(subMux *Mux)) { + for _, r := range mx.tree.routes() { + subMux, ok := r.SubRoutes.(*Mux) + if !ok { + continue + } + fn(subMux) + } +} + +// updateRouteHandler builds the single mux handler that is a chain of the middleware +// stack, as defined by calls to Use(), and the tree router (Mux) itself. After this +// point, no other middlewares can be registered on this Mux's stack. But you can still +// compose additional middlewares via Group()'s or using a chained middleware handler. +func (mx *Mux) updateRouteHandler() { + mx.handler = chain(mx.middlewares, http.HandlerFunc(mx.routeHTTP)) +} + +// methodNotAllowedHandler is a helper function to respond with a 405, +// method not allowed. +func methodNotAllowedHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(405) + w.Write(nil) +} diff --git a/vendor/github.com/go-chi/chi/tree.go b/vendor/github.com/go-chi/chi/tree.go new file mode 100644 index 00000000..8057c528 --- /dev/null +++ b/vendor/github.com/go-chi/chi/tree.go @@ -0,0 +1,866 @@ +package chi + +// Radix tree implementation below is a based on the original work by +// Armon Dadgar in https://github.com/armon/go-radix/blob/master/radix.go +// (MIT licensed). It's been heavily modified for use as a HTTP routing tree. + +import ( + "fmt" + "net/http" + "regexp" + "sort" + "strconv" + "strings" +) + +type methodTyp int + +const ( + mSTUB methodTyp = 1 << iota + mCONNECT + mDELETE + mGET + mHEAD + mOPTIONS + mPATCH + mPOST + mPUT + mTRACE +) + +var mALL = mCONNECT | mDELETE | mGET | mHEAD | + mOPTIONS | mPATCH | mPOST | mPUT | mTRACE + +var methodMap = map[string]methodTyp{ + http.MethodConnect: mCONNECT, + http.MethodDelete: mDELETE, + http.MethodGet: mGET, + http.MethodHead: mHEAD, + http.MethodOptions: mOPTIONS, + http.MethodPatch: mPATCH, + http.MethodPost: mPOST, + http.MethodPut: mPUT, + http.MethodTrace: mTRACE, +} + +// RegisterMethod adds support for custom HTTP method handlers, available +// via Router#Method and Router#MethodFunc +func RegisterMethod(method string) { + if method == "" { + return + } + method = strings.ToUpper(method) + if _, ok := methodMap[method]; ok { + return + } + n := len(methodMap) + if n > strconv.IntSize-2 { + panic(fmt.Sprintf("chi: max number of methods reached (%d)", strconv.IntSize)) + } + mt := methodTyp(2 << n) + methodMap[method] = mt + mALL |= mt +} + +type nodeTyp uint8 + +const ( + ntStatic nodeTyp = iota // /home + ntRegexp // /{id:[0-9]+} + ntParam // /{user} + ntCatchAll // /api/v1/* +) + +type node struct { + // node type: static, regexp, param, catchAll + typ nodeTyp + + // first byte of the prefix + label byte + + // first byte of the child prefix + tail byte + + // prefix is the common prefix we ignore + prefix string + + // regexp matcher for regexp nodes + rex *regexp.Regexp + + // HTTP handler endpoints on the leaf node + endpoints endpoints + + // subroutes on the leaf node + subroutes Routes + + // child nodes should be stored in-order for iteration, + // in groups of the node type. + children [ntCatchAll + 1]nodes +} + +// endpoints is a mapping of http method constants to handlers +// for a given route. +type endpoints map[methodTyp]*endpoint + +type endpoint struct { + // endpoint handler + handler http.Handler + + // pattern is the routing pattern for handler nodes + pattern string + + // parameter keys recorded on handler nodes + paramKeys []string +} + +func (s endpoints) Value(method methodTyp) *endpoint { + mh, ok := s[method] + if !ok { + mh = &endpoint{} + s[method] = mh + } + return mh +} + +func (n *node) InsertRoute(method methodTyp, pattern string, handler http.Handler) *node { + var parent *node + search := pattern + + for { + // Handle key exhaustion + if len(search) == 0 { + // Insert or update the node's leaf handler + n.setEndpoint(method, handler, pattern) + return n + } + + // We're going to be searching for a wild node next, + // in this case, we need to get the tail + var label = search[0] + var segTail byte + var segEndIdx int + var segTyp nodeTyp + var segRexpat string + if label == '{' || label == '*' { + segTyp, _, segRexpat, segTail, _, segEndIdx = patNextSegment(search) + } + + var prefix string + if segTyp == ntRegexp { + prefix = segRexpat + } + + // Look for the edge to attach to + parent = n + n = n.getEdge(segTyp, label, segTail, prefix) + + // No edge, create one + if n == nil { + child := &node{label: label, tail: segTail, prefix: search} + hn := parent.addChild(child, search) + hn.setEndpoint(method, handler, pattern) + + return hn + } + + // Found an edge to match the pattern + + if n.typ > ntStatic { + // We found a param node, trim the param from the search path and continue. + // This param/wild pattern segment would already be on the tree from a previous + // call to addChild when creating a new node. + search = search[segEndIdx:] + continue + } + + // Static nodes fall below here. + // Determine longest prefix of the search key on match. + commonPrefix := longestPrefix(search, n.prefix) + if commonPrefix == len(n.prefix) { + // the common prefix is as long as the current node's prefix we're attempting to insert. + // keep the search going. + search = search[commonPrefix:] + continue + } + + // Split the node + child := &node{ + typ: ntStatic, + prefix: search[:commonPrefix], + } + parent.replaceChild(search[0], segTail, child) + + // Restore the existing node + n.label = n.prefix[commonPrefix] + n.prefix = n.prefix[commonPrefix:] + child.addChild(n, n.prefix) + + // If the new key is a subset, set the method/handler on this node and finish. + search = search[commonPrefix:] + if len(search) == 0 { + child.setEndpoint(method, handler, pattern) + return child + } + + // Create a new edge for the node + subchild := &node{ + typ: ntStatic, + label: search[0], + prefix: search, + } + hn := child.addChild(subchild, search) + hn.setEndpoint(method, handler, pattern) + return hn + } +} + +// addChild appends the new `child` node to the tree using the `pattern` as the trie key. +// For a URL router like chi's, we split the static, param, regexp and wildcard segments +// into different nodes. In addition, addChild will recursively call itself until every +// pattern segment is added to the url pattern tree as individual nodes, depending on type. +func (n *node) addChild(child *node, prefix string) *node { + search := prefix + + // handler leaf node added to the tree is the child. + // this may be overridden later down the flow + hn := child + + // Parse next segment + segTyp, _, segRexpat, segTail, segStartIdx, segEndIdx := patNextSegment(search) + + // Add child depending on next up segment + switch segTyp { + + case ntStatic: + // Search prefix is all static (that is, has no params in path) + // noop + + default: + // Search prefix contains a param, regexp or wildcard + + if segTyp == ntRegexp { + rex, err := regexp.Compile(segRexpat) + if err != nil { + panic(fmt.Sprintf("chi: invalid regexp pattern '%s' in route param", segRexpat)) + } + child.prefix = segRexpat + child.rex = rex + } + + if segStartIdx == 0 { + // Route starts with a param + child.typ = segTyp + + if segTyp == ntCatchAll { + segStartIdx = -1 + } else { + segStartIdx = segEndIdx + } + if segStartIdx < 0 { + segStartIdx = len(search) + } + child.tail = segTail // for params, we set the tail + + if segStartIdx != len(search) { + // add static edge for the remaining part, split the end. + // its not possible to have adjacent param nodes, so its certainly + // going to be a static node next. + + search = search[segStartIdx:] // advance search position + + nn := &node{ + typ: ntStatic, + label: search[0], + prefix: search, + } + hn = child.addChild(nn, search) + } + + } else if segStartIdx > 0 { + // Route has some param + + // starts with a static segment + child.typ = ntStatic + child.prefix = search[:segStartIdx] + child.rex = nil + + // add the param edge node + search = search[segStartIdx:] + + nn := &node{ + typ: segTyp, + label: search[0], + tail: segTail, + } + hn = child.addChild(nn, search) + + } + } + + n.children[child.typ] = append(n.children[child.typ], child) + n.children[child.typ].Sort() + return hn +} + +func (n *node) replaceChild(label, tail byte, child *node) { + for i := 0; i < len(n.children[child.typ]); i++ { + if n.children[child.typ][i].label == label && n.children[child.typ][i].tail == tail { + n.children[child.typ][i] = child + n.children[child.typ][i].label = label + n.children[child.typ][i].tail = tail + return + } + } + panic("chi: replacing missing child") +} + +func (n *node) getEdge(ntyp nodeTyp, label, tail byte, prefix string) *node { + nds := n.children[ntyp] + for i := 0; i < len(nds); i++ { + if nds[i].label == label && nds[i].tail == tail { + if ntyp == ntRegexp && nds[i].prefix != prefix { + continue + } + return nds[i] + } + } + return nil +} + +func (n *node) setEndpoint(method methodTyp, handler http.Handler, pattern string) { + // Set the handler for the method type on the node + if n.endpoints == nil { + n.endpoints = make(endpoints) + } + + paramKeys := patParamKeys(pattern) + + if method&mSTUB == mSTUB { + n.endpoints.Value(mSTUB).handler = handler + } + if method&mALL == mALL { + h := n.endpoints.Value(mALL) + h.handler = handler + h.pattern = pattern + h.paramKeys = paramKeys + for _, m := range methodMap { + h := n.endpoints.Value(m) + h.handler = handler + h.pattern = pattern + h.paramKeys = paramKeys + } + } else { + h := n.endpoints.Value(method) + h.handler = handler + h.pattern = pattern + h.paramKeys = paramKeys + } +} + +func (n *node) FindRoute(rctx *Context, method methodTyp, path string) (*node, endpoints, http.Handler) { + // Reset the context routing pattern and params + rctx.routePattern = "" + rctx.routeParams.Keys = rctx.routeParams.Keys[:0] + rctx.routeParams.Values = rctx.routeParams.Values[:0] + + // Find the routing handlers for the path + rn := n.findRoute(rctx, method, path) + if rn == nil { + return nil, nil, nil + } + + // Record the routing params in the request lifecycle + rctx.URLParams.Keys = append(rctx.URLParams.Keys, rctx.routeParams.Keys...) + rctx.URLParams.Values = append(rctx.URLParams.Values, rctx.routeParams.Values...) + + // Record the routing pattern in the request lifecycle + if rn.endpoints[method].pattern != "" { + rctx.routePattern = rn.endpoints[method].pattern + rctx.RoutePatterns = append(rctx.RoutePatterns, rctx.routePattern) + } + + return rn, rn.endpoints, rn.endpoints[method].handler +} + +// Recursive edge traversal by checking all nodeTyp groups along the way. +// It's like searching through a multi-dimensional radix trie. +func (n *node) findRoute(rctx *Context, method methodTyp, path string) *node { + nn := n + search := path + + for t, nds := range nn.children { + ntyp := nodeTyp(t) + if len(nds) == 0 { + continue + } + + var xn *node + xsearch := search + + var label byte + if search != "" { + label = search[0] + } + + switch ntyp { + case ntStatic: + xn = nds.findEdge(label) + if xn == nil || !strings.HasPrefix(xsearch, xn.prefix) { + continue + } + xsearch = xsearch[len(xn.prefix):] + + case ntParam, ntRegexp: + // short-circuit and return no matching route for empty param values + if xsearch == "" { + continue + } + + // serially loop through each node grouped by the tail delimiter + for idx := 0; idx < len(nds); idx++ { + xn = nds[idx] + + // label for param nodes is the delimiter byte + p := strings.IndexByte(xsearch, xn.tail) + + if p < 0 { + if xn.tail == '/' { + p = len(xsearch) + } else { + continue + } + } else if ntyp == ntRegexp && p == 0 { + continue + } + + if ntyp == ntRegexp && xn.rex != nil { + if !xn.rex.MatchString(xsearch[:p]) { + continue + } + } else if strings.IndexByte(xsearch[:p], '/') != -1 { + // avoid a match across path segments + continue + } + + prevlen := len(rctx.routeParams.Values) + rctx.routeParams.Values = append(rctx.routeParams.Values, xsearch[:p]) + xsearch = xsearch[p:] + + if len(xsearch) == 0 { + if xn.isLeaf() { + h := xn.endpoints[method] + if h != nil && h.handler != nil { + rctx.routeParams.Keys = append(rctx.routeParams.Keys, h.paramKeys...) + return xn + } + + // flag that the routing context found a route, but not a corresponding + // supported method + rctx.methodNotAllowed = true + } + } + + // recursively find the next node on this branch + fin := xn.findRoute(rctx, method, xsearch) + if fin != nil { + return fin + } + + // not found on this branch, reset vars + rctx.routeParams.Values = rctx.routeParams.Values[:prevlen] + xsearch = search + } + + rctx.routeParams.Values = append(rctx.routeParams.Values, "") + + default: + // catch-all nodes + rctx.routeParams.Values = append(rctx.routeParams.Values, search) + xn = nds[0] + xsearch = "" + } + + if xn == nil { + continue + } + + // did we find it yet? + if len(xsearch) == 0 { + if xn.isLeaf() { + h := xn.endpoints[method] + if h != nil && h.handler != nil { + rctx.routeParams.Keys = append(rctx.routeParams.Keys, h.paramKeys...) + return xn + } + + // flag that the routing context found a route, but not a corresponding + // supported method + rctx.methodNotAllowed = true + } + } + + // recursively find the next node.. + fin := xn.findRoute(rctx, method, xsearch) + if fin != nil { + return fin + } + + // Did not find final handler, let's remove the param here if it was set + if xn.typ > ntStatic { + if len(rctx.routeParams.Values) > 0 { + rctx.routeParams.Values = rctx.routeParams.Values[:len(rctx.routeParams.Values)-1] + } + } + + } + + return nil +} + +func (n *node) findEdge(ntyp nodeTyp, label byte) *node { + nds := n.children[ntyp] + num := len(nds) + idx := 0 + + switch ntyp { + case ntStatic, ntParam, ntRegexp: + i, j := 0, num-1 + for i <= j { + idx = i + (j-i)/2 + if label > nds[idx].label { + i = idx + 1 + } else if label < nds[idx].label { + j = idx - 1 + } else { + i = num // breaks cond + } + } + if nds[idx].label != label { + return nil + } + return nds[idx] + + default: // catch all + return nds[idx] + } +} + +func (n *node) isLeaf() bool { + return n.endpoints != nil +} + +func (n *node) findPattern(pattern string) bool { + nn := n + for _, nds := range nn.children { + if len(nds) == 0 { + continue + } + + n = nn.findEdge(nds[0].typ, pattern[0]) + if n == nil { + continue + } + + var idx int + var xpattern string + + switch n.typ { + case ntStatic: + idx = longestPrefix(pattern, n.prefix) + if idx < len(n.prefix) { + continue + } + + case ntParam, ntRegexp: + idx = strings.IndexByte(pattern, '}') + 1 + + case ntCatchAll: + idx = longestPrefix(pattern, "*") + + default: + panic("chi: unknown node type") + } + + xpattern = pattern[idx:] + if len(xpattern) == 0 { + return true + } + + return n.findPattern(xpattern) + } + return false +} + +func (n *node) routes() []Route { + rts := []Route{} + + n.walk(func(eps endpoints, subroutes Routes) bool { + if eps[mSTUB] != nil && eps[mSTUB].handler != nil && subroutes == nil { + return false + } + + // Group methodHandlers by unique patterns + pats := make(map[string]endpoints) + + for mt, h := range eps { + if h.pattern == "" { + continue + } + p, ok := pats[h.pattern] + if !ok { + p = endpoints{} + pats[h.pattern] = p + } + p[mt] = h + } + + for p, mh := range pats { + hs := make(map[string]http.Handler) + if mh[mALL] != nil && mh[mALL].handler != nil { + hs["*"] = mh[mALL].handler + } + + for mt, h := range mh { + if h.handler == nil { + continue + } + m := methodTypString(mt) + if m == "" { + continue + } + hs[m] = h.handler + } + + rt := Route{p, hs, subroutes} + rts = append(rts, rt) + } + + return false + }) + + return rts +} + +func (n *node) walk(fn func(eps endpoints, subroutes Routes) bool) bool { + // Visit the leaf values if any + if (n.endpoints != nil || n.subroutes != nil) && fn(n.endpoints, n.subroutes) { + return true + } + + // Recurse on the children + for _, ns := range n.children { + for _, cn := range ns { + if cn.walk(fn) { + return true + } + } + } + return false +} + +// patNextSegment returns the next segment details from a pattern: +// node type, param key, regexp string, param tail byte, param starting index, param ending index +func patNextSegment(pattern string) (nodeTyp, string, string, byte, int, int) { + ps := strings.Index(pattern, "{") + ws := strings.Index(pattern, "*") + + if ps < 0 && ws < 0 { + return ntStatic, "", "", 0, 0, len(pattern) // we return the entire thing + } + + // Sanity check + if ps >= 0 && ws >= 0 && ws < ps { + panic("chi: wildcard '*' must be the last pattern in a route, otherwise use a '{param}'") + } + + var tail byte = '/' // Default endpoint tail to / byte + + if ps >= 0 { + // Param/Regexp pattern is next + nt := ntParam + + // Read to closing } taking into account opens and closes in curl count (cc) + cc := 0 + pe := ps + for i, c := range pattern[ps:] { + if c == '{' { + cc++ + } else if c == '}' { + cc-- + if cc == 0 { + pe = ps + i + break + } + } + } + if pe == ps { + panic("chi: route param closing delimiter '}' is missing") + } + + key := pattern[ps+1 : pe] + pe++ // set end to next position + + if pe < len(pattern) { + tail = pattern[pe] + } + + var rexpat string + if idx := strings.Index(key, ":"); idx >= 0 { + nt = ntRegexp + rexpat = key[idx+1:] + key = key[:idx] + } + + if len(rexpat) > 0 { + if rexpat[0] != '^' { + rexpat = "^" + rexpat + } + if rexpat[len(rexpat)-1] != '$' { + rexpat += "$" + } + } + + return nt, key, rexpat, tail, ps, pe + } + + // Wildcard pattern as finale + if ws < len(pattern)-1 { + panic("chi: wildcard '*' must be the last value in a route. trim trailing text or use a '{param}' instead") + } + return ntCatchAll, "*", "", 0, ws, len(pattern) +} + +func patParamKeys(pattern string) []string { + pat := pattern + paramKeys := []string{} + for { + ptyp, paramKey, _, _, _, e := patNextSegment(pat) + if ptyp == ntStatic { + return paramKeys + } + for i := 0; i < len(paramKeys); i++ { + if paramKeys[i] == paramKey { + panic(fmt.Sprintf("chi: routing pattern '%s' contains duplicate param key, '%s'", pattern, paramKey)) + } + } + paramKeys = append(paramKeys, paramKey) + pat = pat[e:] + } +} + +// longestPrefix finds the length of the shared prefix +// of two strings +func longestPrefix(k1, k2 string) int { + max := len(k1) + if l := len(k2); l < max { + max = l + } + var i int + for i = 0; i < max; i++ { + if k1[i] != k2[i] { + break + } + } + return i +} + +func methodTypString(method methodTyp) string { + for s, t := range methodMap { + if method == t { + return s + } + } + return "" +} + +type nodes []*node + +// Sort the list of nodes by label +func (ns nodes) Sort() { sort.Sort(ns); ns.tailSort() } +func (ns nodes) Len() int { return len(ns) } +func (ns nodes) Swap(i, j int) { ns[i], ns[j] = ns[j], ns[i] } +func (ns nodes) Less(i, j int) bool { return ns[i].label < ns[j].label } + +// tailSort pushes nodes with '/' as the tail to the end of the list for param nodes. +// The list order determines the traversal order. +func (ns nodes) tailSort() { + for i := len(ns) - 1; i >= 0; i-- { + if ns[i].typ > ntStatic && ns[i].tail == '/' { + ns.Swap(i, len(ns)-1) + return + } + } +} + +func (ns nodes) findEdge(label byte) *node { + num := len(ns) + idx := 0 + i, j := 0, num-1 + for i <= j { + idx = i + (j-i)/2 + if label > ns[idx].label { + i = idx + 1 + } else if label < ns[idx].label { + j = idx - 1 + } else { + i = num // breaks cond + } + } + if ns[idx].label != label { + return nil + } + return ns[idx] +} + +// Route describes the details of a routing handler. +// Handlers map key is an HTTP method +type Route struct { + Pattern string + Handlers map[string]http.Handler + SubRoutes Routes +} + +// WalkFunc is the type of the function called for each method and route visited by Walk. +type WalkFunc func(method string, route string, handler http.Handler, middlewares ...func(http.Handler) http.Handler) error + +// Walk walks any router tree that implements Routes interface. +func Walk(r Routes, walkFn WalkFunc) error { + return walk(r, walkFn, "") +} + +func walk(r Routes, walkFn WalkFunc, parentRoute string, parentMw ...func(http.Handler) http.Handler) error { + for _, route := range r.Routes() { + mws := make([]func(http.Handler) http.Handler, len(parentMw)) + copy(mws, parentMw) + mws = append(mws, r.Middlewares()...) + + if route.SubRoutes != nil { + if err := walk(route.SubRoutes, walkFn, parentRoute+route.Pattern, mws...); err != nil { + return err + } + continue + } + + for method, handler := range route.Handlers { + if method == "*" { + // Ignore a "catchAll" method, since we pass down all the specific methods for each route. + continue + } + + fullRoute := parentRoute + route.Pattern + fullRoute = strings.Replace(fullRoute, "/*/", "/", -1) + + if chain, ok := handler.(*ChainHandler); ok { + if err := walkFn(method, fullRoute, chain.Endpoint, append(mws, chain.Middlewares...)...); err != nil { + return err + } + } else { + if err := walkFn(method, fullRoute, handler, mws...); err != nil { + return err + } + } + } + } + + return nil +} diff --git a/vendor/github.com/go-chi/chi/v5/.gitignore b/vendor/github.com/go-chi/chi/v5/.gitignore new file mode 100644 index 00000000..ba22c99a --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/.gitignore @@ -0,0 +1,3 @@ +.idea +*.sw? +.vscode diff --git a/vendor/github.com/go-chi/chi/v5/CHANGELOG.md b/vendor/github.com/go-chi/chi/v5/CHANGELOG.md new file mode 100644 index 00000000..f6eb7e6e --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/CHANGELOG.md @@ -0,0 +1,331 @@ +# Changelog + +## v5.0.10 (2023-07-13) + +- Fixed small edge case in tests of v5.0.9 for older Go versions +- History of changes: see https://github.com/go-chi/chi/compare/v5.0.8...v5.0.10 + + +## v5.0.9 (2023-07-13) + +- History of changes: see https://github.com/go-chi/chi/compare/v5.0.8...v5.0.9 + + +## v5.0.8 (2022-12-07) + +- History of changes: see https://github.com/go-chi/chi/compare/v5.0.7...v5.0.8 + + +## v5.0.7 (2021-11-18) + +- History of changes: see https://github.com/go-chi/chi/compare/v5.0.6...v5.0.7 + + +## v5.0.6 (2021-11-15) + +- History of changes: see https://github.com/go-chi/chi/compare/v5.0.5...v5.0.6 + + +## v5.0.5 (2021-10-27) + +- History of changes: see https://github.com/go-chi/chi/compare/v5.0.4...v5.0.5 + + +## v5.0.4 (2021-08-29) + +- History of changes: see https://github.com/go-chi/chi/compare/v5.0.3...v5.0.4 + + +## v5.0.3 (2021-04-29) + +- History of changes: see https://github.com/go-chi/chi/compare/v5.0.2...v5.0.3 + + +## v5.0.2 (2021-03-25) + +- History of changes: see https://github.com/go-chi/chi/compare/v5.0.1...v5.0.2 + + +## v5.0.1 (2021-03-10) + +- Small improvements +- History of changes: see https://github.com/go-chi/chi/compare/v5.0.0...v5.0.1 + + +## v5.0.0 (2021-02-27) + +- chi v5, `github.com/go-chi/chi/v5` introduces the adoption of Go's SIV to adhere to the current state-of-the-tools in Go. +- chi v1.5.x did not work out as planned, as the Go tooling is too powerful and chi's adoption is too wide. + The most responsible thing to do for everyone's benefit is to just release v5 with SIV, so I present to you all, + chi v5 at `github.com/go-chi/chi/v5`. I hope someday the developer experience and ergonomics I've been seeking + will still come to fruition in some form, see https://github.com/golang/go/issues/44550 +- History of changes: see https://github.com/go-chi/chi/compare/v1.5.4...v5.0.0 + + +## v1.5.4 (2021-02-27) + +- Undo prior retraction in v1.5.3 as we prepare for v5.0.0 release +- History of changes: see https://github.com/go-chi/chi/compare/v1.5.3...v1.5.4 + + +## v1.5.3 (2021-02-21) + +- Update go.mod to go 1.16 with new retract directive marking all versions without prior go.mod support +- History of changes: see https://github.com/go-chi/chi/compare/v1.5.2...v1.5.3 + + +## v1.5.2 (2021-02-10) + +- Reverting allocation optimization as a precaution as go test -race fails. +- Minor improvements, see history below +- History of changes: see https://github.com/go-chi/chi/compare/v1.5.1...v1.5.2 + + +## v1.5.1 (2020-12-06) + +- Performance improvement: removing 1 allocation by foregoing context.WithValue, thank you @bouk for + your contribution (https://github.com/go-chi/chi/pull/555). Note: new benchmarks posted in README. +- `middleware.CleanPath`: new middleware that clean's request path of double slashes +- deprecate & remove `chi.ServerBaseContext` in favour of stdlib `http.Server#BaseContext` +- plus other tiny improvements, see full commit history below +- History of changes: see https://github.com/go-chi/chi/compare/v4.1.2...v1.5.1 + + +## v1.5.0 (2020-11-12) - now with go.mod support + +`chi` dates back to 2016 with it's original implementation as one of the first routers to adopt the newly introduced +context.Context api to the stdlib -- set out to design a router that is faster, more modular and simpler than anything +else out there -- while not introducing any custom handler types or dependencies. Today, `chi` still has zero dependencies, +and in many ways is future proofed from changes, given it's minimal nature. Between versions, chi's iterations have been very +incremental, with the architecture and api being the same today as it was originally designed in 2016. For this reason it +makes chi a pretty easy project to maintain, as well thanks to the many amazing community contributions over the years +to who all help make chi better (total of 86 contributors to date -- thanks all!). + +Chi has been a labour of love, art and engineering, with the goals to offer beautiful ergonomics, flexibility, performance +and simplicity when building HTTP services with Go. I've strived to keep the router very minimal in surface area / code size, +and always improving the code wherever possible -- and as of today the `chi` package is just 1082 lines of code (not counting +middlewares, which are all optional). As well, I don't have the exact metrics, but from my analysis and email exchanges from +companies and developers, chi is used by thousands of projects around the world -- thank you all as there is no better form of +joy for me than to have art I had started be helpful and enjoyed by others. And of course I use chi in all of my own projects too :) + +For me, the aesthetics of chi's code and usage are very important. With the introduction of Go's module support +(which I'm a big fan of), chi's past versioning scheme choice to v2, v3 and v4 would mean I'd require the import path +of "github.com/go-chi/chi/v4", leading to the lengthy discussion at https://github.com/go-chi/chi/issues/462. +Haha, to some, you may be scratching your head why I've spent > 1 year stalling to adopt "/vXX" convention in the import +path -- which isn't horrible in general -- but for chi, I'm unable to accept it as I strive for perfection in it's API design, +aesthetics and simplicity. It just doesn't feel good to me given chi's simple nature -- I do not foresee a "v5" or "v6", +and upgrading between versions in the future will also be just incremental. + +I do understand versioning is a part of the API design as well, which is why the solution for a while has been to "do nothing", +as Go supports both old and new import paths with/out go.mod. However, now that Go module support has had time to iron out kinks and +is adopted everywhere, it's time for chi to get with the times. Luckily, I've discovered a path forward that will make me happy, +while also not breaking anyone's app who adopted a prior versioning from tags in v2/v3/v4. I've made an experimental release of +v1.5.0 with go.mod silently, and tested it with new and old projects, to ensure the developer experience is preserved, and it's +largely unnoticed. Fortunately, Go's toolchain will check the tags of a repo and consider the "latest" tag the one with go.mod. +However, you can still request a specific older tag such as v4.1.2, and everything will "just work". But new users can just +`go get github.com/go-chi/chi` or `go get github.com/go-chi/chi@latest` and they will get the latest version which contains +go.mod support, which is v1.5.0+. `chi` will not change very much over the years, just like it hasn't changed much from 4 years ago. +Therefore, we will stay on v1.x from here on, starting from v1.5.0. Any breaking changes will bump a "minor" release and +backwards-compatible improvements/fixes will bump a "tiny" release. + +For existing projects who want to upgrade to the latest go.mod version, run: `go get -u github.com/go-chi/chi@v1.5.0`, +which will get you on the go.mod version line (as Go's mod cache may still remember v4.x). Brand new systems can run +`go get -u github.com/go-chi/chi` or `go get -u github.com/go-chi/chi@latest` to install chi, which will install v1.5.0+ +built with go.mod support. + +My apologies to the developers who will disagree with the decisions above, but, hope you'll try it and see it's a very +minor request which is backwards compatible and won't break your existing installations. + +Cheers all, happy coding! + + +--- + + +## v4.1.2 (2020-06-02) + +- fix that handles MethodNotAllowed with path variables, thank you @caseyhadden for your contribution +- fix to replace nested wildcards correctly in RoutePattern, thank you @@unmultimedio for your contribution +- History of changes: see https://github.com/go-chi/chi/compare/v4.1.1...v4.1.2 + + +## v4.1.1 (2020-04-16) + +- fix for issue https://github.com/go-chi/chi/issues/411 which allows for overlapping regexp + route to the correct handler through a recursive tree search, thanks to @Jahaja for the PR/fix! +- new middleware.RouteHeaders as a simple router for request headers with wildcard support +- History of changes: see https://github.com/go-chi/chi/compare/v4.1.0...v4.1.1 + + +## v4.1.0 (2020-04-1) + +- middleware.LogEntry: Write method on interface now passes the response header + and an extra interface type useful for custom logger implementations. +- middleware.WrapResponseWriter: minor fix +- middleware.Recoverer: a bit prettier +- History of changes: see https://github.com/go-chi/chi/compare/v4.0.4...v4.1.0 + +## v4.0.4 (2020-03-24) + +- middleware.Recoverer: new pretty stack trace printing (https://github.com/go-chi/chi/pull/496) +- a few minor improvements and fixes +- History of changes: see https://github.com/go-chi/chi/compare/v4.0.3...v4.0.4 + + +## v4.0.3 (2020-01-09) + +- core: fix regexp routing to include default value when param is not matched +- middleware: rewrite of middleware.Compress +- middleware: suppress http.ErrAbortHandler in middleware.Recoverer +- History of changes: see https://github.com/go-chi/chi/compare/v4.0.2...v4.0.3 + + +## v4.0.2 (2019-02-26) + +- Minor fixes +- History of changes: see https://github.com/go-chi/chi/compare/v4.0.1...v4.0.2 + + +## v4.0.1 (2019-01-21) + +- Fixes issue with compress middleware: #382 #385 +- History of changes: see https://github.com/go-chi/chi/compare/v4.0.0...v4.0.1 + + +## v4.0.0 (2019-01-10) + +- chi v4 requires Go 1.10.3+ (or Go 1.9.7+) - we have deprecated support for Go 1.7 and 1.8 +- router: respond with 404 on router with no routes (#362) +- router: additional check to ensure wildcard is at the end of a url pattern (#333) +- middleware: deprecate use of http.CloseNotifier (#347) +- middleware: fix RedirectSlashes to include query params on redirect (#334) +- History of changes: see https://github.com/go-chi/chi/compare/v3.3.4...v4.0.0 + + +## v3.3.4 (2019-01-07) + +- Minor middleware improvements. No changes to core library/router. Moving v3 into its +- own branch as a version of chi for Go 1.7, 1.8, 1.9, 1.10, 1.11 +- History of changes: see https://github.com/go-chi/chi/compare/v3.3.3...v3.3.4 + + +## v3.3.3 (2018-08-27) + +- Minor release +- See https://github.com/go-chi/chi/compare/v3.3.2...v3.3.3 + + +## v3.3.2 (2017-12-22) + +- Support to route trailing slashes on mounted sub-routers (#281) +- middleware: new `ContentCharset` to check matching charsets. Thank you + @csucu for your community contribution! + + +## v3.3.1 (2017-11-20) + +- middleware: new `AllowContentType` handler for explicit whitelist of accepted request Content-Types +- middleware: new `SetHeader` handler for short-hand middleware to set a response header key/value +- Minor bug fixes + + +## v3.3.0 (2017-10-10) + +- New chi.RegisterMethod(method) to add support for custom HTTP methods, see _examples/custom-method for usage +- Deprecated LINK and UNLINK methods from the default list, please use `chi.RegisterMethod("LINK")` and `chi.RegisterMethod("UNLINK")` in an `init()` function + + +## v3.2.1 (2017-08-31) + +- Add new `Match(rctx *Context, method, path string) bool` method to `Routes` interface + and `Mux`. Match searches the mux's routing tree for a handler that matches the method/path +- Add new `RouteMethod` to `*Context` +- Add new `Routes` pointer to `*Context` +- Add new `middleware.GetHead` to route missing HEAD requests to GET handler +- Updated benchmarks (see README) + + +## v3.1.5 (2017-08-02) + +- Setup golint and go vet for the project +- As per golint, we've redefined `func ServerBaseContext(h http.Handler, baseCtx context.Context) http.Handler` + to `func ServerBaseContext(baseCtx context.Context, h http.Handler) http.Handler` + + +## v3.1.0 (2017-07-10) + +- Fix a few minor issues after v3 release +- Move `docgen` sub-pkg to https://github.com/go-chi/docgen +- Move `render` sub-pkg to https://github.com/go-chi/render +- Add new `URLFormat` handler to chi/middleware sub-pkg to make working with url mime + suffixes easier, ie. parsing `/articles/1.json` and `/articles/1.xml`. See comments in + https://github.com/go-chi/chi/blob/master/middleware/url_format.go for example usage. + + +## v3.0.0 (2017-06-21) + +- Major update to chi library with many exciting updates, but also some *breaking changes* +- URL parameter syntax changed from `/:id` to `/{id}` for even more flexible routing, such as + `/articles/{month}-{day}-{year}-{slug}`, `/articles/{id}`, and `/articles/{id}.{ext}` on the + same router +- Support for regexp for routing patterns, in the form of `/{paramKey:regExp}` for example: + `r.Get("/articles/{name:[a-z]+}", h)` and `chi.URLParam(r, "name")` +- Add `Method` and `MethodFunc` to `chi.Router` to allow routing definitions such as + `r.Method("GET", "/", h)` which provides a cleaner interface for custom handlers like + in `_examples/custom-handler` +- Deprecating `mux#FileServer` helper function. Instead, we encourage users to create their + own using file handler with the stdlib, see `_examples/fileserver` for an example +- Add support for LINK/UNLINK http methods via `r.Method()` and `r.MethodFunc()` +- Moved the chi project to its own organization, to allow chi-related community packages to + be easily discovered and supported, at: https://github.com/go-chi +- *NOTE:* please update your import paths to `"github.com/go-chi/chi"` +- *NOTE:* chi v2 is still available at https://github.com/go-chi/chi/tree/v2 + + +## v2.1.0 (2017-03-30) + +- Minor improvements and update to the chi core library +- Introduced a brand new `chi/render` sub-package to complete the story of building + APIs to offer a pattern for managing well-defined request / response payloads. Please + check out the updated `_examples/rest` example for how it works. +- Added `MethodNotAllowed(h http.HandlerFunc)` to chi.Router interface + + +## v2.0.0 (2017-01-06) + +- After many months of v2 being in an RC state with many companies and users running it in + production, the inclusion of some improvements to the middlewares, we are very pleased to + announce v2.0.0 of chi. + + +## v2.0.0-rc1 (2016-07-26) + +- Huge update! chi v2 is a large refactor targeting Go 1.7+. As of Go 1.7, the popular + community `"net/context"` package has been included in the standard library as `"context"` and + utilized by `"net/http"` and `http.Request` to managing deadlines, cancelation signals and other + request-scoped values. We're very excited about the new context addition and are proud to + introduce chi v2, a minimal and powerful routing package for building large HTTP services, + with zero external dependencies. Chi focuses on idiomatic design and encourages the use of + stdlib HTTP handlers and middlwares. +- chi v2 deprecates its `chi.Handler` interface and requires `http.Handler` or `http.HandlerFunc` +- chi v2 stores URL routing parameters and patterns in the standard request context: `r.Context()` +- chi v2 lower-level routing context is accessible by `chi.RouteContext(r.Context()) *chi.Context`, + which provides direct access to URL routing parameters, the routing path and the matching + routing patterns. +- Users upgrading from chi v1 to v2, need to: + 1. Update the old chi.Handler signature, `func(ctx context.Context, w http.ResponseWriter, r *http.Request)` to + the standard http.Handler: `func(w http.ResponseWriter, r *http.Request)` + 2. Use `chi.URLParam(r *http.Request, paramKey string) string` + or `URLParamFromCtx(ctx context.Context, paramKey string) string` to access a url parameter value + + +## v1.0.0 (2016-07-01) + +- Released chi v1 stable https://github.com/go-chi/chi/tree/v1.0.0 for Go 1.6 and older. + + +## v0.9.0 (2016-03-31) + +- Reuse context objects via sync.Pool for zero-allocation routing [#33](https://github.com/go-chi/chi/pull/33) +- BREAKING NOTE: due to subtle API changes, previously `chi.URLParams(ctx)["id"]` used to access url parameters + has changed to: `chi.URLParam(ctx, "id")` diff --git a/vendor/github.com/go-chi/chi/v5/CONTRIBUTING.md b/vendor/github.com/go-chi/chi/v5/CONTRIBUTING.md new file mode 100644 index 00000000..c0ac2dfe --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/CONTRIBUTING.md @@ -0,0 +1,31 @@ +# Contributing + +## Prerequisites + +1. [Install Go][go-install]. +2. Download the sources and switch the working directory: + + ```bash + go get -u -d github.com/go-chi/chi + cd $GOPATH/src/github.com/go-chi/chi + ``` + +## Submitting a Pull Request + +A typical workflow is: + +1. [Fork the repository.][fork] [This tip maybe also helpful.][go-fork-tip] +2. [Create a topic branch.][branch] +3. Add tests for your change. +4. Run `go test`. If your tests pass, return to the step 3. +5. Implement the change and ensure the steps from the previous step pass. +6. Run `goimports -w .`, to ensure the new code conforms to Go formatting guideline. +7. [Add, commit and push your changes.][git-help] +8. [Submit a pull request.][pull-req] + +[go-install]: https://golang.org/doc/install +[go-fork-tip]: http://blog.campoy.cat/2014/03/github-and-go-forking-pull-requests-and.html +[fork]: https://help.github.com/articles/fork-a-repo +[branch]: http://learn.github.com/p/branching.html +[git-help]: https://guides.github.com +[pull-req]: https://help.github.com/articles/using-pull-requests diff --git a/vendor/github.com/go-chi/chi/v5/LICENSE b/vendor/github.com/go-chi/chi/v5/LICENSE new file mode 100644 index 00000000..d99f02ff --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2015-present Peter Kieltyka (https://github.com/pkieltyka), Google Inc. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/go-chi/chi/v5/Makefile b/vendor/github.com/go-chi/chi/v5/Makefile new file mode 100644 index 00000000..e0f18c7d --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/Makefile @@ -0,0 +1,22 @@ +.PHONY: all +all: + @echo "**********************************************************" + @echo "** chi build tool **" + @echo "**********************************************************" + + +.PHONY: test +test: + go clean -testcache && $(MAKE) test-router && $(MAKE) test-middleware + +.PHONY: test-router +test-router: + go test -race -v . + +.PHONY: test-middleware +test-middleware: + go test -race -v ./middleware + +.PHONY: docs +docs: + npx docsify-cli serve ./docs diff --git a/vendor/github.com/go-chi/chi/v5/README.md b/vendor/github.com/go-chi/chi/v5/README.md new file mode 100644 index 00000000..718e373f --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/README.md @@ -0,0 +1,500 @@ +# chi + + +[![GoDoc Widget]][GoDoc] + +`chi` is a lightweight, idiomatic and composable router for building Go HTTP services. It's +especially good at helping you write large REST API services that are kept maintainable as your +project grows and changes. `chi` is built on the new `context` package introduced in Go 1.7 to +handle signaling, cancelation and request-scoped values across a handler chain. + +The focus of the project has been to seek out an elegant and comfortable design for writing +REST API servers, written during the development of the Pressly API service that powers our +public API service, which in turn powers all of our client-side applications. + +The key considerations of chi's design are: project structure, maintainability, standard http +handlers (stdlib-only), developer productivity, and deconstructing a large system into many small +parts. The core router `github.com/go-chi/chi` is quite small (less than 1000 LOC), but we've also +included some useful/optional subpackages: [middleware](/middleware), [render](https://github.com/go-chi/render) +and [docgen](https://github.com/go-chi/docgen). We hope you enjoy it too! + +## Install + +`go get -u github.com/go-chi/chi/v5` + + +## Features + +* **Lightweight** - cloc'd in ~1000 LOC for the chi router +* **Fast** - yes, see [benchmarks](#benchmarks) +* **100% compatible with net/http** - use any http or middleware pkg in the ecosystem that is also compatible with `net/http` +* **Designed for modular/composable APIs** - middlewares, inline middlewares, route groups and sub-router mounting +* **Context control** - built on new `context` package, providing value chaining, cancellations and timeouts +* **Robust** - in production at Pressly, Cloudflare, Heroku, 99Designs, and many others (see [discussion](https://github.com/go-chi/chi/issues/91)) +* **Doc generation** - `docgen` auto-generates routing documentation from your source to JSON or Markdown +* **Go.mod support** - as of v5, go.mod support (see [CHANGELOG](https://github.com/go-chi/chi/blob/master/CHANGELOG.md)) +* **No external dependencies** - plain ol' Go stdlib + net/http + + +## Examples + +See [_examples/](https://github.com/go-chi/chi/blob/master/_examples/) for a variety of examples. + + +**As easy as:** + +```go +package main + +import ( + "net/http" + + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" +) + +func main() { + r := chi.NewRouter() + r.Use(middleware.Logger) + r.Get("/", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("welcome")) + }) + http.ListenAndServe(":3000", r) +} +``` + +**REST Preview:** + +Here is a little preview of how routing looks like with chi. Also take a look at the generated routing docs +in JSON ([routes.json](https://github.com/go-chi/chi/blob/master/_examples/rest/routes.json)) and in +Markdown ([routes.md](https://github.com/go-chi/chi/blob/master/_examples/rest/routes.md)). + +I highly recommend reading the source of the [examples](https://github.com/go-chi/chi/blob/master/_examples/) listed +above, they will show you all the features of chi and serve as a good form of documentation. + +```go +import ( + //... + "context" + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" +) + +func main() { + r := chi.NewRouter() + + // A good base middleware stack + r.Use(middleware.RequestID) + r.Use(middleware.RealIP) + r.Use(middleware.Logger) + r.Use(middleware.Recoverer) + + // Set a timeout value on the request context (ctx), that will signal + // through ctx.Done() that the request has timed out and further + // processing should be stopped. + r.Use(middleware.Timeout(60 * time.Second)) + + r.Get("/", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("hi")) + }) + + // RESTy routes for "articles" resource + r.Route("/articles", func(r chi.Router) { + r.With(paginate).Get("/", listArticles) // GET /articles + r.With(paginate).Get("/{month}-{day}-{year}", listArticlesByDate) // GET /articles/01-16-2017 + + r.Post("/", createArticle) // POST /articles + r.Get("/search", searchArticles) // GET /articles/search + + // Regexp url parameters: + r.Get("/{articleSlug:[a-z-]+}", getArticleBySlug) // GET /articles/home-is-toronto + + // Subrouters: + r.Route("/{articleID}", func(r chi.Router) { + r.Use(ArticleCtx) + r.Get("/", getArticle) // GET /articles/123 + r.Put("/", updateArticle) // PUT /articles/123 + r.Delete("/", deleteArticle) // DELETE /articles/123 + }) + }) + + // Mount the admin sub-router + r.Mount("/admin", adminRouter()) + + http.ListenAndServe(":3333", r) +} + +func ArticleCtx(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + articleID := chi.URLParam(r, "articleID") + article, err := dbGetArticle(articleID) + if err != nil { + http.Error(w, http.StatusText(404), 404) + return + } + ctx := context.WithValue(r.Context(), "article", article) + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +func getArticle(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + article, ok := ctx.Value("article").(*Article) + if !ok { + http.Error(w, http.StatusText(422), 422) + return + } + w.Write([]byte(fmt.Sprintf("title:%s", article.Title))) +} + +// A completely separate router for administrator routes +func adminRouter() http.Handler { + r := chi.NewRouter() + r.Use(AdminOnly) + r.Get("/", adminIndex) + r.Get("/accounts", adminListAccounts) + return r +} + +func AdminOnly(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + perm, ok := ctx.Value("acl.permission").(YourPermissionType) + if !ok || !perm.IsAdmin() { + http.Error(w, http.StatusText(403), 403) + return + } + next.ServeHTTP(w, r) + }) +} +``` + + +## Router interface + +chi's router is based on a kind of [Patricia Radix trie](https://en.wikipedia.org/wiki/Radix_tree). +The router is fully compatible with `net/http`. + +Built on top of the tree is the `Router` interface: + +```go +// Router consisting of the core routing methods used by chi's Mux, +// using only the standard net/http. +type Router interface { + http.Handler + Routes + + // Use appends one or more middlewares onto the Router stack. + Use(middlewares ...func(http.Handler) http.Handler) + + // With adds inline middlewares for an endpoint handler. + With(middlewares ...func(http.Handler) http.Handler) Router + + // Group adds a new inline-Router along the current routing + // path, with a fresh middleware stack for the inline-Router. + Group(fn func(r Router)) Router + + // Route mounts a sub-Router along a `pattern`` string. + Route(pattern string, fn func(r Router)) Router + + // Mount attaches another http.Handler along ./pattern/* + Mount(pattern string, h http.Handler) + + // Handle and HandleFunc adds routes for `pattern` that matches + // all HTTP methods. + Handle(pattern string, h http.Handler) + HandleFunc(pattern string, h http.HandlerFunc) + + // Method and MethodFunc adds routes for `pattern` that matches + // the `method` HTTP method. + Method(method, pattern string, h http.Handler) + MethodFunc(method, pattern string, h http.HandlerFunc) + + // HTTP-method routing along `pattern` + Connect(pattern string, h http.HandlerFunc) + Delete(pattern string, h http.HandlerFunc) + Get(pattern string, h http.HandlerFunc) + Head(pattern string, h http.HandlerFunc) + Options(pattern string, h http.HandlerFunc) + Patch(pattern string, h http.HandlerFunc) + Post(pattern string, h http.HandlerFunc) + Put(pattern string, h http.HandlerFunc) + Trace(pattern string, h http.HandlerFunc) + + // NotFound defines a handler to respond whenever a route could + // not be found. + NotFound(h http.HandlerFunc) + + // MethodNotAllowed defines a handler to respond whenever a method is + // not allowed. + MethodNotAllowed(h http.HandlerFunc) +} + +// Routes interface adds two methods for router traversal, which is also +// used by the github.com/go-chi/docgen package to generate documentation for Routers. +type Routes interface { + // Routes returns the routing tree in an easily traversable structure. + Routes() []Route + + // Middlewares returns the list of middlewares in use by the router. + Middlewares() Middlewares + + // Match searches the routing tree for a handler that matches + // the method/path - similar to routing a http request, but without + // executing the handler thereafter. + Match(rctx *Context, method, path string) bool +} +``` + +Each routing method accepts a URL `pattern` and chain of `handlers`. The URL pattern +supports named params (ie. `/users/{userID}`) and wildcards (ie. `/admin/*`). URL parameters +can be fetched at runtime by calling `chi.URLParam(r, "userID")` for named parameters +and `chi.URLParam(r, "*")` for a wildcard parameter. + + +### Middleware handlers + +chi's middlewares are just stdlib net/http middleware handlers. There is nothing special +about them, which means the router and all the tooling is designed to be compatible and +friendly with any middleware in the community. This offers much better extensibility and reuse +of packages and is at the heart of chi's purpose. + +Here is an example of a standard net/http middleware where we assign a context key `"user"` +the value of `"123"`. This middleware sets a hypothetical user identifier on the request +context and calls the next handler in the chain. + +```go +// HTTP middleware setting a value on the request context +func MyMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // create new context from `r` request context, and assign key `"user"` + // to value of `"123"` + ctx := context.WithValue(r.Context(), "user", "123") + + // call the next handler in the chain, passing the response writer and + // the updated request object with the new context value. + // + // note: context.Context values are nested, so any previously set + // values will be accessible as well, and the new `"user"` key + // will be accessible from this point forward. + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} +``` + + +### Request handlers + +chi uses standard net/http request handlers. This little snippet is an example of a http.Handler +func that reads a user identifier from the request context - hypothetically, identifying +the user sending an authenticated request, validated+set by a previous middleware handler. + +```go +// HTTP handler accessing data from the request context. +func MyRequestHandler(w http.ResponseWriter, r *http.Request) { + // here we read from the request context and fetch out `"user"` key set in + // the MyMiddleware example above. + user := r.Context().Value("user").(string) + + // respond to the client + w.Write([]byte(fmt.Sprintf("hi %s", user))) +} +``` + + +### URL parameters + +chi's router parses and stores URL parameters right onto the request context. Here is +an example of how to access URL params in your net/http handlers. And of course, middlewares +are able to access the same information. + +```go +// HTTP handler accessing the url routing parameters. +func MyRequestHandler(w http.ResponseWriter, r *http.Request) { + // fetch the url parameter `"userID"` from the request of a matching + // routing pattern. An example routing pattern could be: /users/{userID} + userID := chi.URLParam(r, "userID") + + // fetch `"key"` from the request context + ctx := r.Context() + key := ctx.Value("key").(string) + + // respond to the client + w.Write([]byte(fmt.Sprintf("hi %v, %v", userID, key))) +} +``` + + +## Middlewares + +chi comes equipped with an optional `middleware` package, providing a suite of standard +`net/http` middlewares. Please note, any middleware in the ecosystem that is also compatible +with `net/http` can be used with chi's mux. + +### Core middlewares + +---------------------------------------------------------------------------------------------------- +| chi/middleware Handler | description | +| :--------------------- | :---------------------------------------------------------------------- | +| [AllowContentEncoding] | Enforces a whitelist of request Content-Encoding headers | +| [AllowContentType] | Explicit whitelist of accepted request Content-Types | +| [BasicAuth] | Basic HTTP authentication | +| [Compress] | Gzip compression for clients that accept compressed responses | +| [ContentCharset] | Ensure charset for Content-Type request headers | +| [CleanPath] | Clean double slashes from request path | +| [GetHead] | Automatically route undefined HEAD requests to GET handlers | +| [Heartbeat] | Monitoring endpoint to check the servers pulse | +| [Logger] | Logs the start and end of each request with the elapsed processing time | +| [NoCache] | Sets response headers to prevent clients from caching | +| [Profiler] | Easily attach net/http/pprof to your routers | +| [RealIP] | Sets a http.Request's RemoteAddr to either X-Real-IP or X-Forwarded-For | +| [Recoverer] | Gracefully absorb panics and prints the stack trace | +| [RequestID] | Injects a request ID into the context of each request | +| [RedirectSlashes] | Redirect slashes on routing paths | +| [RouteHeaders] | Route handling for request headers | +| [SetHeader] | Short-hand middleware to set a response header key/value | +| [StripSlashes] | Strip slashes on routing paths | +| [Throttle] | Puts a ceiling on the number of concurrent requests | +| [Timeout] | Signals to the request context when the timeout deadline is reached | +| [URLFormat] | Parse extension from url and put it on request context | +| [WithValue] | Short-hand middleware to set a key/value on the request context | +---------------------------------------------------------------------------------------------------- + +[AllowContentEncoding]: https://pkg.go.dev/github.com/go-chi/chi/middleware#AllowContentEncoding +[AllowContentType]: https://pkg.go.dev/github.com/go-chi/chi/middleware#AllowContentType +[BasicAuth]: https://pkg.go.dev/github.com/go-chi/chi/middleware#BasicAuth +[Compress]: https://pkg.go.dev/github.com/go-chi/chi/middleware#Compress +[ContentCharset]: https://pkg.go.dev/github.com/go-chi/chi/middleware#ContentCharset +[CleanPath]: https://pkg.go.dev/github.com/go-chi/chi/middleware#CleanPath +[GetHead]: https://pkg.go.dev/github.com/go-chi/chi/middleware#GetHead +[GetReqID]: https://pkg.go.dev/github.com/go-chi/chi/middleware#GetReqID +[Heartbeat]: https://pkg.go.dev/github.com/go-chi/chi/middleware#Heartbeat +[Logger]: https://pkg.go.dev/github.com/go-chi/chi/middleware#Logger +[NoCache]: https://pkg.go.dev/github.com/go-chi/chi/middleware#NoCache +[Profiler]: https://pkg.go.dev/github.com/go-chi/chi/middleware#Profiler +[RealIP]: https://pkg.go.dev/github.com/go-chi/chi/middleware#RealIP +[Recoverer]: https://pkg.go.dev/github.com/go-chi/chi/middleware#Recoverer +[RedirectSlashes]: https://pkg.go.dev/github.com/go-chi/chi/middleware#RedirectSlashes +[RequestLogger]: https://pkg.go.dev/github.com/go-chi/chi/middleware#RequestLogger +[RequestID]: https://pkg.go.dev/github.com/go-chi/chi/middleware#RequestID +[RouteHeaders]: https://pkg.go.dev/github.com/go-chi/chi/middleware#RouteHeaders +[SetHeader]: https://pkg.go.dev/github.com/go-chi/chi/middleware#SetHeader +[StripSlashes]: https://pkg.go.dev/github.com/go-chi/chi/middleware#StripSlashes +[Throttle]: https://pkg.go.dev/github.com/go-chi/chi/middleware#Throttle +[ThrottleBacklog]: https://pkg.go.dev/github.com/go-chi/chi/middleware#ThrottleBacklog +[ThrottleWithOpts]: https://pkg.go.dev/github.com/go-chi/chi/middleware#ThrottleWithOpts +[Timeout]: https://pkg.go.dev/github.com/go-chi/chi/middleware#Timeout +[URLFormat]: https://pkg.go.dev/github.com/go-chi/chi/middleware#URLFormat +[WithLogEntry]: https://pkg.go.dev/github.com/go-chi/chi/middleware#WithLogEntry +[WithValue]: https://pkg.go.dev/github.com/go-chi/chi/middleware#WithValue +[Compressor]: https://pkg.go.dev/github.com/go-chi/chi/middleware#Compressor +[DefaultLogFormatter]: https://pkg.go.dev/github.com/go-chi/chi/middleware#DefaultLogFormatter +[EncoderFunc]: https://pkg.go.dev/github.com/go-chi/chi/middleware#EncoderFunc +[HeaderRoute]: https://pkg.go.dev/github.com/go-chi/chi/middleware#HeaderRoute +[HeaderRouter]: https://pkg.go.dev/github.com/go-chi/chi/middleware#HeaderRouter +[LogEntry]: https://pkg.go.dev/github.com/go-chi/chi/middleware#LogEntry +[LogFormatter]: https://pkg.go.dev/github.com/go-chi/chi/middleware#LogFormatter +[LoggerInterface]: https://pkg.go.dev/github.com/go-chi/chi/middleware#LoggerInterface +[ThrottleOpts]: https://pkg.go.dev/github.com/go-chi/chi/middleware#ThrottleOpts +[WrapResponseWriter]: https://pkg.go.dev/github.com/go-chi/chi/middleware#WrapResponseWriter + +### Extra middlewares & packages + +Please see https://github.com/go-chi for additional packages. + +-------------------------------------------------------------------------------------------------------------------- +| package | description | +|:---------------------------------------------------|:------------------------------------------------------------- +| [cors](https://github.com/go-chi/cors) | Cross-origin resource sharing (CORS) | +| [docgen](https://github.com/go-chi/docgen) | Print chi.Router routes at runtime | +| [jwtauth](https://github.com/go-chi/jwtauth) | JWT authentication | +| [hostrouter](https://github.com/go-chi/hostrouter) | Domain/host based request routing | +| [httplog](https://github.com/go-chi/httplog) | Small but powerful structured HTTP request logging | +| [httprate](https://github.com/go-chi/httprate) | HTTP request rate limiter | +| [httptracer](https://github.com/go-chi/httptracer) | HTTP request performance tracing library | +| [httpvcr](https://github.com/go-chi/httpvcr) | Write deterministic tests for external sources | +| [stampede](https://github.com/go-chi/stampede) | HTTP request coalescer | +-------------------------------------------------------------------------------------------------------------------- + + +## context? + +`context` is a tiny pkg that provides simple interface to signal context across call stacks +and goroutines. It was originally written by [Sameer Ajmani](https://github.com/Sajmani) +and is available in stdlib since go1.7. + +Learn more at https://blog.golang.org/context + +and.. +* Docs: https://golang.org/pkg/context +* Source: https://github.com/golang/go/tree/master/src/context + + +## Benchmarks + +The benchmark suite: https://github.com/pkieltyka/go-http-routing-benchmark + +Results as of Nov 29, 2020 with Go 1.15.5 on Linux AMD 3950x + +```shell +BenchmarkChi_Param 3075895 384 ns/op 400 B/op 2 allocs/op +BenchmarkChi_Param5 2116603 566 ns/op 400 B/op 2 allocs/op +BenchmarkChi_Param20 964117 1227 ns/op 400 B/op 2 allocs/op +BenchmarkChi_ParamWrite 2863413 420 ns/op 400 B/op 2 allocs/op +BenchmarkChi_GithubStatic 3045488 395 ns/op 400 B/op 2 allocs/op +BenchmarkChi_GithubParam 2204115 540 ns/op 400 B/op 2 allocs/op +BenchmarkChi_GithubAll 10000 113811 ns/op 81203 B/op 406 allocs/op +BenchmarkChi_GPlusStatic 3337485 359 ns/op 400 B/op 2 allocs/op +BenchmarkChi_GPlusParam 2825853 423 ns/op 400 B/op 2 allocs/op +BenchmarkChi_GPlus2Params 2471697 483 ns/op 400 B/op 2 allocs/op +BenchmarkChi_GPlusAll 194220 5950 ns/op 5200 B/op 26 allocs/op +BenchmarkChi_ParseStatic 3365324 356 ns/op 400 B/op 2 allocs/op +BenchmarkChi_ParseParam 2976614 404 ns/op 400 B/op 2 allocs/op +BenchmarkChi_Parse2Params 2638084 439 ns/op 400 B/op 2 allocs/op +BenchmarkChi_ParseAll 109567 11295 ns/op 10400 B/op 52 allocs/op +BenchmarkChi_StaticAll 16846 71308 ns/op 62802 B/op 314 allocs/op +``` + +Comparison with other routers: https://gist.github.com/pkieltyka/123032f12052520aaccab752bd3e78cc + +NOTE: the allocs in the benchmark above are from the calls to http.Request's +`WithContext(context.Context)` method that clones the http.Request, sets the `Context()` +on the duplicated (alloc'd) request and returns it the new request object. This is just +how setting context on a request in Go works. + + +## Credits + +* Carl Jackson for https://github.com/zenazn/goji + * Parts of chi's thinking comes from goji, and chi's middleware package + sources from goji. +* Armon Dadgar for https://github.com/armon/go-radix +* Contributions: [@VojtechVitek](https://github.com/VojtechVitek) + +We'll be more than happy to see [your contributions](./CONTRIBUTING.md)! + + +## Beyond REST + +chi is just a http router that lets you decompose request handling into many smaller layers. +Many companies use chi to write REST services for their public APIs. But, REST is just a convention +for managing state via HTTP, and there's a lot of other pieces required to write a complete client-server +system or network of microservices. + +Looking beyond REST, I also recommend some newer works in the field: +* [webrpc](https://github.com/webrpc/webrpc) - Web-focused RPC client+server framework with code-gen +* [gRPC](https://github.com/grpc/grpc-go) - Google's RPC framework via protobufs +* [graphql](https://github.com/99designs/gqlgen) - Declarative query language +* [NATS](https://nats.io) - lightweight pub-sub + + +## License + +Copyright (c) 2015-present [Peter Kieltyka](https://github.com/pkieltyka) + +Licensed under [MIT License](./LICENSE) + +[GoDoc]: https://pkg.go.dev/github.com/go-chi/chi?tab=versions +[GoDoc Widget]: https://godoc.org/github.com/go-chi/chi?status.svg +[Travis]: https://travis-ci.org/go-chi/chi +[Travis Widget]: https://travis-ci.org/go-chi/chi.svg?branch=master diff --git a/vendor/github.com/go-chi/chi/v5/chain.go b/vendor/github.com/go-chi/chi/v5/chain.go new file mode 100644 index 00000000..a2278414 --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/chain.go @@ -0,0 +1,49 @@ +package chi + +import "net/http" + +// Chain returns a Middlewares type from a slice of middleware handlers. +func Chain(middlewares ...func(http.Handler) http.Handler) Middlewares { + return Middlewares(middlewares) +} + +// Handler builds and returns a http.Handler from the chain of middlewares, +// with `h http.Handler` as the final handler. +func (mws Middlewares) Handler(h http.Handler) http.Handler { + return &ChainHandler{h, chain(mws, h), mws} +} + +// HandlerFunc builds and returns a http.Handler from the chain of middlewares, +// with `h http.Handler` as the final handler. +func (mws Middlewares) HandlerFunc(h http.HandlerFunc) http.Handler { + return &ChainHandler{h, chain(mws, h), mws} +} + +// ChainHandler is a http.Handler with support for handler composition and +// execution. +type ChainHandler struct { + Endpoint http.Handler + chain http.Handler + Middlewares Middlewares +} + +func (c *ChainHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + c.chain.ServeHTTP(w, r) +} + +// chain builds a http.Handler composed of an inline middleware stack and endpoint +// handler in the order they are passed. +func chain(middlewares []func(http.Handler) http.Handler, endpoint http.Handler) http.Handler { + // Return ahead of time if there aren't any middlewares for the chain + if len(middlewares) == 0 { + return endpoint + } + + // Wrap the end handler with the middleware chain + h := middlewares[len(middlewares)-1](endpoint) + for i := len(middlewares) - 2; i >= 0; i-- { + h = middlewares[i](h) + } + + return h +} diff --git a/vendor/github.com/go-chi/chi/v5/chi.go b/vendor/github.com/go-chi/chi/v5/chi.go new file mode 100644 index 00000000..a1691bbe --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/chi.go @@ -0,0 +1,134 @@ +// Package chi is a small, idiomatic and composable router for building HTTP services. +// +// chi requires Go 1.14 or newer. +// +// Example: +// +// package main +// +// import ( +// "net/http" +// +// "github.com/go-chi/chi/v5" +// "github.com/go-chi/chi/v5/middleware" +// ) +// +// func main() { +// r := chi.NewRouter() +// r.Use(middleware.Logger) +// r.Use(middleware.Recoverer) +// +// r.Get("/", func(w http.ResponseWriter, r *http.Request) { +// w.Write([]byte("root.")) +// }) +// +// http.ListenAndServe(":3333", r) +// } +// +// See github.com/go-chi/chi/_examples/ for more in-depth examples. +// +// URL patterns allow for easy matching of path components in HTTP +// requests. The matching components can then be accessed using +// chi.URLParam(). All patterns must begin with a slash. +// +// A simple named placeholder {name} matches any sequence of characters +// up to the next / or the end of the URL. Trailing slashes on paths must +// be handled explicitly. +// +// A placeholder with a name followed by a colon allows a regular +// expression match, for example {number:\\d+}. The regular expression +// syntax is Go's normal regexp RE2 syntax, except that regular expressions +// including { or } are not supported, and / will never be +// matched. An anonymous regexp pattern is allowed, using an empty string +// before the colon in the placeholder, such as {:\\d+} +// +// The special placeholder of asterisk matches the rest of the requested +// URL. Any trailing characters in the pattern are ignored. This is the only +// placeholder which will match / characters. +// +// Examples: +// +// "/user/{name}" matches "/user/jsmith" but not "/user/jsmith/info" or "/user/jsmith/" +// "/user/{name}/info" matches "/user/jsmith/info" +// "/page/*" matches "/page/intro/latest" +// "/page/{other}/index" also matches "/page/intro/latest" +// "/date/{yyyy:\\d\\d\\d\\d}/{mm:\\d\\d}/{dd:\\d\\d}" matches "/date/2017/04/01" +package chi + +import "net/http" + +// NewRouter returns a new Mux object that implements the Router interface. +func NewRouter() *Mux { + return NewMux() +} + +// Router consisting of the core routing methods used by chi's Mux, +// using only the standard net/http. +type Router interface { + http.Handler + Routes + + // Use appends one or more middlewares onto the Router stack. + Use(middlewares ...func(http.Handler) http.Handler) + + // With adds inline middlewares for an endpoint handler. + With(middlewares ...func(http.Handler) http.Handler) Router + + // Group adds a new inline-Router along the current routing + // path, with a fresh middleware stack for the inline-Router. + Group(fn func(r Router)) Router + + // Route mounts a sub-Router along a `pattern`` string. + Route(pattern string, fn func(r Router)) Router + + // Mount attaches another http.Handler along ./pattern/* + Mount(pattern string, h http.Handler) + + // Handle and HandleFunc adds routes for `pattern` that matches + // all HTTP methods. + Handle(pattern string, h http.Handler) + HandleFunc(pattern string, h http.HandlerFunc) + + // Method and MethodFunc adds routes for `pattern` that matches + // the `method` HTTP method. + Method(method, pattern string, h http.Handler) + MethodFunc(method, pattern string, h http.HandlerFunc) + + // HTTP-method routing along `pattern` + Connect(pattern string, h http.HandlerFunc) + Delete(pattern string, h http.HandlerFunc) + Get(pattern string, h http.HandlerFunc) + Head(pattern string, h http.HandlerFunc) + Options(pattern string, h http.HandlerFunc) + Patch(pattern string, h http.HandlerFunc) + Post(pattern string, h http.HandlerFunc) + Put(pattern string, h http.HandlerFunc) + Trace(pattern string, h http.HandlerFunc) + + // NotFound defines a handler to respond whenever a route could + // not be found. + NotFound(h http.HandlerFunc) + + // MethodNotAllowed defines a handler to respond whenever a method is + // not allowed. + MethodNotAllowed(h http.HandlerFunc) +} + +// Routes interface adds two methods for router traversal, which is also +// used by the `docgen` subpackage to generation documentation for Routers. +type Routes interface { + // Routes returns the routing tree in an easily traversable structure. + Routes() []Route + + // Middlewares returns the list of middlewares in use by the router. + Middlewares() Middlewares + + // Match searches the routing tree for a handler that matches + // the method/path - similar to routing a http request, but without + // executing the handler thereafter. + Match(rctx *Context, method, path string) bool +} + +// Middlewares type is a slice of standard middleware handlers with methods +// to compose middleware chains and http.Handler's. +type Middlewares []func(http.Handler) http.Handler diff --git a/vendor/github.com/go-chi/chi/v5/context.go b/vendor/github.com/go-chi/chi/v5/context.go new file mode 100644 index 00000000..88f8e221 --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/context.go @@ -0,0 +1,160 @@ +package chi + +import ( + "context" + "net/http" + "strings" +) + +// URLParam returns the url parameter from a http.Request object. +func URLParam(r *http.Request, key string) string { + if rctx := RouteContext(r.Context()); rctx != nil { + return rctx.URLParam(key) + } + return "" +} + +// URLParamFromCtx returns the url parameter from a http.Request Context. +func URLParamFromCtx(ctx context.Context, key string) string { + if rctx := RouteContext(ctx); rctx != nil { + return rctx.URLParam(key) + } + return "" +} + +// RouteContext returns chi's routing Context object from a +// http.Request Context. +func RouteContext(ctx context.Context) *Context { + val, _ := ctx.Value(RouteCtxKey).(*Context) + return val +} + +// NewRouteContext returns a new routing Context object. +func NewRouteContext() *Context { + return &Context{} +} + +var ( + // RouteCtxKey is the context.Context key to store the request context. + RouteCtxKey = &contextKey{"RouteContext"} +) + +// Context is the default routing context set on the root node of a +// request context to track route patterns, URL parameters and +// an optional routing path. +type Context struct { + Routes Routes + + // parentCtx is the parent of this one, for using Context as a + // context.Context directly. This is an optimization that saves + // 1 allocation. + parentCtx context.Context + + // Routing path/method override used during the route search. + // See Mux#routeHTTP method. + RoutePath string + RouteMethod string + + // URLParams are the stack of routeParams captured during the + // routing lifecycle across a stack of sub-routers. + URLParams RouteParams + + // Route parameters matched for the current sub-router. It is + // intentionally unexported so it cant be tampered. + routeParams RouteParams + + // The endpoint routing pattern that matched the request URI path + // or `RoutePath` of the current sub-router. This value will update + // during the lifecycle of a request passing through a stack of + // sub-routers. + routePattern string + + // Routing pattern stack throughout the lifecycle of the request, + // across all connected routers. It is a record of all matching + // patterns across a stack of sub-routers. + RoutePatterns []string + + // methodNotAllowed hint + methodNotAllowed bool + methodsAllowed []methodTyp // allowed methods in case of a 405 +} + +// Reset a routing context to its initial state. +func (x *Context) Reset() { + x.Routes = nil + x.RoutePath = "" + x.RouteMethod = "" + x.RoutePatterns = x.RoutePatterns[:0] + x.URLParams.Keys = x.URLParams.Keys[:0] + x.URLParams.Values = x.URLParams.Values[:0] + + x.routePattern = "" + x.routeParams.Keys = x.routeParams.Keys[:0] + x.routeParams.Values = x.routeParams.Values[:0] + x.methodNotAllowed = false + x.parentCtx = nil +} + +// URLParam returns the corresponding URL parameter value from the request +// routing context. +func (x *Context) URLParam(key string) string { + for k := len(x.URLParams.Keys) - 1; k >= 0; k-- { + if x.URLParams.Keys[k] == key { + return x.URLParams.Values[k] + } + } + return "" +} + +// RoutePattern builds the routing pattern string for the particular +// request, at the particular point during routing. This means, the value +// will change throughout the execution of a request in a router. That is +// why its advised to only use this value after calling the next handler. +// +// For example, +// +// func Instrument(next http.Handler) http.Handler { +// return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +// next.ServeHTTP(w, r) +// routePattern := chi.RouteContext(r.Context()).RoutePattern() +// measure(w, r, routePattern) +// }) +// } +func (x *Context) RoutePattern() string { + routePattern := strings.Join(x.RoutePatterns, "") + routePattern = replaceWildcards(routePattern) + routePattern = strings.TrimSuffix(routePattern, "//") + routePattern = strings.TrimSuffix(routePattern, "/") + return routePattern +} + +// replaceWildcards takes a route pattern and recursively replaces all +// occurrences of "/*/" to "/". +func replaceWildcards(p string) string { + if strings.Contains(p, "/*/") { + return replaceWildcards(strings.Replace(p, "/*/", "/", -1)) + } + return p +} + +// RouteParams is a structure to track URL routing parameters efficiently. +type RouteParams struct { + Keys, Values []string +} + +// Add will append a URL parameter to the end of the route param +func (s *RouteParams) Add(key, value string) { + s.Keys = append(s.Keys, key) + s.Values = append(s.Values, value) +} + +// contextKey is a value for use with context.WithValue. It's used as +// a pointer so it fits in an interface{} without allocation. This technique +// for defining context keys was copied from Go 1.7's new use of context in net/http. +type contextKey struct { + name string +} + +func (k *contextKey) String() string { + return "chi context value " + k.name +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/basic_auth.go b/vendor/github.com/go-chi/chi/v5/middleware/basic_auth.go new file mode 100644 index 00000000..a546c9e9 --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/basic_auth.go @@ -0,0 +1,33 @@ +package middleware + +import ( + "crypto/subtle" + "fmt" + "net/http" +) + +// BasicAuth implements a simple middleware handler for adding basic http auth to a route. +func BasicAuth(realm string, creds map[string]string) func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + user, pass, ok := r.BasicAuth() + if !ok { + basicAuthFailed(w, realm) + return + } + + credPass, credUserOk := creds[user] + if !credUserOk || subtle.ConstantTimeCompare([]byte(pass), []byte(credPass)) != 1 { + basicAuthFailed(w, realm) + return + } + + next.ServeHTTP(w, r) + }) + } +} + +func basicAuthFailed(w http.ResponseWriter, realm string) { + w.Header().Add("WWW-Authenticate", fmt.Sprintf(`Basic realm="%s"`, realm)) + w.WriteHeader(http.StatusUnauthorized) +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/clean_path.go b/vendor/github.com/go-chi/chi/v5/middleware/clean_path.go new file mode 100644 index 00000000..adeba429 --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/clean_path.go @@ -0,0 +1,28 @@ +package middleware + +import ( + "net/http" + "path" + + "github.com/go-chi/chi/v5" +) + +// CleanPath middleware will clean out double slash mistakes from a user's request path. +// For example, if a user requests /users//1 or //users////1 will both be treated as: /users/1 +func CleanPath(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rctx := chi.RouteContext(r.Context()) + + routePath := rctx.RoutePath + if routePath == "" { + if r.URL.RawPath != "" { + routePath = r.URL.RawPath + } else { + routePath = r.URL.Path + } + rctx.RoutePath = path.Clean(routePath) + } + + next.ServeHTTP(w, r) + }) +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/compress.go b/vendor/github.com/go-chi/chi/v5/middleware/compress.go new file mode 100644 index 00000000..773d47a1 --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/compress.go @@ -0,0 +1,403 @@ +package middleware + +import ( + "bufio" + "compress/flate" + "compress/gzip" + "errors" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "strings" + "sync" +) + +var defaultCompressibleContentTypes = []string{ + "text/html", + "text/css", + "text/plain", + "text/javascript", + "application/javascript", + "application/x-javascript", + "application/json", + "application/atom+xml", + "application/rss+xml", + "image/svg+xml", +} + +// Compress is a middleware that compresses response +// body of a given content types to a data format based +// on Accept-Encoding request header. It uses a given +// compression level. +// +// NOTE: make sure to set the Content-Type header on your response +// otherwise this middleware will not compress the response body. For ex, in +// your handler you should set w.Header().Set("Content-Type", http.DetectContentType(yourBody)) +// or set it manually. +// +// Passing a compression level of 5 is sensible value +func Compress(level int, types ...string) func(next http.Handler) http.Handler { + compressor := NewCompressor(level, types...) + return compressor.Handler +} + +// Compressor represents a set of encoding configurations. +type Compressor struct { + // The mapping of encoder names to encoder functions. + encoders map[string]EncoderFunc + // The mapping of pooled encoders to pools. + pooledEncoders map[string]*sync.Pool + // The set of content types allowed to be compressed. + allowedTypes map[string]struct{} + allowedWildcards map[string]struct{} + // The list of encoders in order of decreasing precedence. + encodingPrecedence []string + level int // The compression level. +} + +// NewCompressor creates a new Compressor that will handle encoding responses. +// +// The level should be one of the ones defined in the flate package. +// The types are the content types that are allowed to be compressed. +func NewCompressor(level int, types ...string) *Compressor { + // If types are provided, set those as the allowed types. If none are + // provided, use the default list. + allowedTypes := make(map[string]struct{}) + allowedWildcards := make(map[string]struct{}) + if len(types) > 0 { + for _, t := range types { + if strings.Contains(strings.TrimSuffix(t, "/*"), "*") { + panic(fmt.Sprintf("middleware/compress: Unsupported content-type wildcard pattern '%s'. Only '/*' supported", t)) + } + if strings.HasSuffix(t, "/*") { + allowedWildcards[strings.TrimSuffix(t, "/*")] = struct{}{} + } else { + allowedTypes[t] = struct{}{} + } + } + } else { + for _, t := range defaultCompressibleContentTypes { + allowedTypes[t] = struct{}{} + } + } + + c := &Compressor{ + level: level, + encoders: make(map[string]EncoderFunc), + pooledEncoders: make(map[string]*sync.Pool), + allowedTypes: allowedTypes, + allowedWildcards: allowedWildcards, + } + + // Set the default encoders. The precedence order uses the reverse + // ordering that the encoders were added. This means adding new encoders + // will move them to the front of the order. + // + // TODO: + // lzma: Opera. + // sdch: Chrome, Android. Gzip output + dictionary header. + // br: Brotli, see https://github.com/go-chi/chi/pull/326 + + // HTTP 1.1 "deflate" (RFC 2616) stands for DEFLATE data (RFC 1951) + // wrapped with zlib (RFC 1950). The zlib wrapper uses Adler-32 + // checksum compared to CRC-32 used in "gzip" and thus is faster. + // + // But.. some old browsers (MSIE, Safari 5.1) incorrectly expect + // raw DEFLATE data only, without the mentioned zlib wrapper. + // Because of this major confusion, most modern browsers try it + // both ways, first looking for zlib headers. + // Quote by Mark Adler: http://stackoverflow.com/a/9186091/385548 + // + // The list of browsers having problems is quite big, see: + // http://zoompf.com/blog/2012/02/lose-the-wait-http-compression + // https://web.archive.org/web/20120321182910/http://www.vervestudios.co/projects/compression-tests/results + // + // That's why we prefer gzip over deflate. It's just more reliable + // and not significantly slower than deflate. + c.SetEncoder("deflate", encoderDeflate) + + // TODO: Exception for old MSIE browsers that can't handle non-HTML? + // https://zoompf.com/blog/2012/02/lose-the-wait-http-compression + c.SetEncoder("gzip", encoderGzip) + + // NOTE: Not implemented, intentionally: + // case "compress": // LZW. Deprecated. + // case "bzip2": // Too slow on-the-fly. + // case "zopfli": // Too slow on-the-fly. + // case "xz": // Too slow on-the-fly. + return c +} + +// SetEncoder can be used to set the implementation of a compression algorithm. +// +// The encoding should be a standardised identifier. See: +// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Encoding +// +// For example, add the Brotli algorithm: +// +// import brotli_enc "gopkg.in/kothar/brotli-go.v0/enc" +// +// compressor := middleware.NewCompressor(5, "text/html") +// compressor.SetEncoder("br", func(w io.Writer, level int) io.Writer { +// params := brotli_enc.NewBrotliParams() +// params.SetQuality(level) +// return brotli_enc.NewBrotliWriter(params, w) +// }) +func (c *Compressor) SetEncoder(encoding string, fn EncoderFunc) { + encoding = strings.ToLower(encoding) + if encoding == "" { + panic("the encoding can not be empty") + } + if fn == nil { + panic("attempted to set a nil encoder function") + } + + // If we are adding a new encoder that is already registered, we have to + // clear that one out first. + if _, ok := c.pooledEncoders[encoding]; ok { + delete(c.pooledEncoders, encoding) + } + if _, ok := c.encoders[encoding]; ok { + delete(c.encoders, encoding) + } + + // If the encoder supports Resetting (IoReseterWriter), then it can be pooled. + encoder := fn(ioutil.Discard, c.level) + if encoder != nil { + if _, ok := encoder.(ioResetterWriter); ok { + pool := &sync.Pool{ + New: func() interface{} { + return fn(ioutil.Discard, c.level) + }, + } + c.pooledEncoders[encoding] = pool + } + } + // If the encoder is not in the pooledEncoders, add it to the normal encoders. + if _, ok := c.pooledEncoders[encoding]; !ok { + c.encoders[encoding] = fn + } + + for i, v := range c.encodingPrecedence { + if v == encoding { + c.encodingPrecedence = append(c.encodingPrecedence[:i], c.encodingPrecedence[i+1:]...) + } + } + + c.encodingPrecedence = append([]string{encoding}, c.encodingPrecedence...) +} + +// Handler returns a new middleware that will compress the response based on the +// current Compressor. +func (c *Compressor) Handler(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + encoder, encoding, cleanup := c.selectEncoder(r.Header, w) + + cw := &compressResponseWriter{ + ResponseWriter: w, + w: w, + contentTypes: c.allowedTypes, + contentWildcards: c.allowedWildcards, + encoding: encoding, + compressable: false, // determined in post-handler + } + if encoder != nil { + cw.w = encoder + } + // Re-add the encoder to the pool if applicable. + defer cleanup() + defer cw.Close() + + next.ServeHTTP(cw, r) + }) +} + +// selectEncoder returns the encoder, the name of the encoder, and a closer function. +func (c *Compressor) selectEncoder(h http.Header, w io.Writer) (io.Writer, string, func()) { + header := h.Get("Accept-Encoding") + + // Parse the names of all accepted algorithms from the header. + accepted := strings.Split(strings.ToLower(header), ",") + + // Find supported encoder by accepted list by precedence + for _, name := range c.encodingPrecedence { + if matchAcceptEncoding(accepted, name) { + if pool, ok := c.pooledEncoders[name]; ok { + encoder := pool.Get().(ioResetterWriter) + cleanup := func() { + pool.Put(encoder) + } + encoder.Reset(w) + return encoder, name, cleanup + + } + if fn, ok := c.encoders[name]; ok { + return fn(w, c.level), name, func() {} + } + } + + } + + // No encoder found to match the accepted encoding + return nil, "", func() {} +} + +func matchAcceptEncoding(accepted []string, encoding string) bool { + for _, v := range accepted { + if strings.Contains(v, encoding) { + return true + } + } + return false +} + +// An EncoderFunc is a function that wraps the provided io.Writer with a +// streaming compression algorithm and returns it. +// +// In case of failure, the function should return nil. +type EncoderFunc func(w io.Writer, level int) io.Writer + +// Interface for types that allow resetting io.Writers. +type ioResetterWriter interface { + io.Writer + Reset(w io.Writer) +} + +type compressResponseWriter struct { + http.ResponseWriter + + // The streaming encoder writer to be used if there is one. Otherwise, + // this is just the normal writer. + w io.Writer + contentTypes map[string]struct{} + contentWildcards map[string]struct{} + encoding string + wroteHeader bool + compressable bool +} + +func (cw *compressResponseWriter) isCompressable() bool { + // Parse the first part of the Content-Type response header. + contentType := cw.Header().Get("Content-Type") + if idx := strings.Index(contentType, ";"); idx >= 0 { + contentType = contentType[0:idx] + } + + // Is the content type compressible? + if _, ok := cw.contentTypes[contentType]; ok { + return true + } + if idx := strings.Index(contentType, "/"); idx > 0 { + contentType = contentType[0:idx] + _, ok := cw.contentWildcards[contentType] + return ok + } + return false +} + +func (cw *compressResponseWriter) WriteHeader(code int) { + if cw.wroteHeader { + cw.ResponseWriter.WriteHeader(code) // Allow multiple calls to propagate. + return + } + cw.wroteHeader = true + defer cw.ResponseWriter.WriteHeader(code) + + // Already compressed data? + if cw.Header().Get("Content-Encoding") != "" { + return + } + + if !cw.isCompressable() { + cw.compressable = false + return + } + + if cw.encoding != "" { + cw.compressable = true + cw.Header().Set("Content-Encoding", cw.encoding) + cw.Header().Add("Vary", "Accept-Encoding") + + // The content-length after compression is unknown + cw.Header().Del("Content-Length") + } +} + +func (cw *compressResponseWriter) Write(p []byte) (int, error) { + if !cw.wroteHeader { + cw.WriteHeader(http.StatusOK) + } + + return cw.writer().Write(p) +} + +func (cw *compressResponseWriter) writer() io.Writer { + if cw.compressable { + return cw.w + } else { + return cw.ResponseWriter + } +} + +type compressFlusher interface { + Flush() error +} + +func (cw *compressResponseWriter) Flush() { + if f, ok := cw.writer().(http.Flusher); ok { + f.Flush() + } + // If the underlying writer has a compression flush signature, + // call this Flush() method instead + if f, ok := cw.writer().(compressFlusher); ok { + f.Flush() + + // Also flush the underlying response writer + if f, ok := cw.ResponseWriter.(http.Flusher); ok { + f.Flush() + } + } +} + +func (cw *compressResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + if hj, ok := cw.writer().(http.Hijacker); ok { + return hj.Hijack() + } + return nil, nil, errors.New("chi/middleware: http.Hijacker is unavailable on the writer") +} + +func (cw *compressResponseWriter) Push(target string, opts *http.PushOptions) error { + if ps, ok := cw.writer().(http.Pusher); ok { + return ps.Push(target, opts) + } + return errors.New("chi/middleware: http.Pusher is unavailable on the writer") +} + +func (cw *compressResponseWriter) Close() error { + if c, ok := cw.writer().(io.WriteCloser); ok { + return c.Close() + } + return errors.New("chi/middleware: io.WriteCloser is unavailable on the writer") +} + +func (cw *compressResponseWriter) Unwrap() http.ResponseWriter { + return cw.ResponseWriter +} + +func encoderGzip(w io.Writer, level int) io.Writer { + gw, err := gzip.NewWriterLevel(w, level) + if err != nil { + return nil + } + return gw +} + +func encoderDeflate(w io.Writer, level int) io.Writer { + dw, err := flate.NewWriter(w, level) + if err != nil { + return nil + } + return dw +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/content_charset.go b/vendor/github.com/go-chi/chi/v5/middleware/content_charset.go new file mode 100644 index 00000000..07b5ce6f --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/content_charset.go @@ -0,0 +1,51 @@ +package middleware + +import ( + "net/http" + "strings" +) + +// ContentCharset generates a handler that writes a 415 Unsupported Media Type response if none of the charsets match. +// An empty charset will allow requests with no Content-Type header or no specified charset. +func ContentCharset(charsets ...string) func(next http.Handler) http.Handler { + for i, c := range charsets { + charsets[i] = strings.ToLower(c) + } + + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !contentEncoding(r.Header.Get("Content-Type"), charsets...) { + w.WriteHeader(http.StatusUnsupportedMediaType) + return + } + + next.ServeHTTP(w, r) + }) + } +} + +// Check the content encoding against a list of acceptable values. +func contentEncoding(ce string, charsets ...string) bool { + _, ce = split(strings.ToLower(ce), ";") + _, ce = split(ce, "charset=") + ce, _ = split(ce, ";") + for _, c := range charsets { + if ce == c { + return true + } + } + + return false +} + +// Split a string in two parts, cleaning any whitespace. +func split(str, sep string) (string, string) { + var a, b string + var parts = strings.SplitN(str, sep, 2) + a = strings.TrimSpace(parts[0]) + if len(parts) == 2 { + b = strings.TrimSpace(parts[1]) + } + + return a, b +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/content_encoding.go b/vendor/github.com/go-chi/chi/v5/middleware/content_encoding.go new file mode 100644 index 00000000..e0b9ccc0 --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/content_encoding.go @@ -0,0 +1,34 @@ +package middleware + +import ( + "net/http" + "strings" +) + +// AllowContentEncoding enforces a whitelist of request Content-Encoding otherwise responds +// with a 415 Unsupported Media Type status. +func AllowContentEncoding(contentEncoding ...string) func(next http.Handler) http.Handler { + allowedEncodings := make(map[string]struct{}, len(contentEncoding)) + for _, encoding := range contentEncoding { + allowedEncodings[strings.TrimSpace(strings.ToLower(encoding))] = struct{}{} + } + return func(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + requestEncodings := r.Header["Content-Encoding"] + // skip check for empty content body or no Content-Encoding + if r.ContentLength == 0 { + next.ServeHTTP(w, r) + return + } + // All encodings in the request must be allowed + for _, encoding := range requestEncodings { + if _, ok := allowedEncodings[strings.TrimSpace(strings.ToLower(encoding))]; !ok { + w.WriteHeader(http.StatusUnsupportedMediaType) + return + } + } + next.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) + } +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/content_type.go b/vendor/github.com/go-chi/chi/v5/middleware/content_type.go new file mode 100644 index 00000000..023978fa --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/content_type.go @@ -0,0 +1,49 @@ +package middleware + +import ( + "net/http" + "strings" +) + +// SetHeader is a convenience handler to set a response header key/value +func SetHeader(key, value string) func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(key, value) + next.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) + } +} + +// AllowContentType enforces a whitelist of request Content-Types otherwise responds +// with a 415 Unsupported Media Type status. +func AllowContentType(contentTypes ...string) func(next http.Handler) http.Handler { + allowedContentTypes := make(map[string]struct{}, len(contentTypes)) + for _, ctype := range contentTypes { + allowedContentTypes[strings.TrimSpace(strings.ToLower(ctype))] = struct{}{} + } + + return func(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + if r.ContentLength == 0 { + // skip check for empty content body + next.ServeHTTP(w, r) + return + } + + s := strings.ToLower(strings.TrimSpace(r.Header.Get("Content-Type"))) + if i := strings.Index(s, ";"); i > -1 { + s = s[0:i] + } + + if _, ok := allowedContentTypes[s]; ok { + next.ServeHTTP(w, r) + return + } + + w.WriteHeader(http.StatusUnsupportedMediaType) + } + return http.HandlerFunc(fn) + } +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/get_head.go b/vendor/github.com/go-chi/chi/v5/middleware/get_head.go new file mode 100644 index 00000000..d4606d8b --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/get_head.go @@ -0,0 +1,39 @@ +package middleware + +import ( + "net/http" + + "github.com/go-chi/chi/v5" +) + +// GetHead automatically route undefined HEAD requests to GET handlers. +func GetHead(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "HEAD" { + rctx := chi.RouteContext(r.Context()) + routePath := rctx.RoutePath + if routePath == "" { + if r.URL.RawPath != "" { + routePath = r.URL.RawPath + } else { + routePath = r.URL.Path + } + } + + // Temporary routing context to look-ahead before routing the request + tctx := chi.NewRouteContext() + + // Attempt to find a HEAD handler for the routing path, if not found, traverse + // the router as through its a GET route, but proceed with the request + // with the HEAD method. + if !rctx.Routes.Match(tctx, "HEAD", routePath) { + rctx.RouteMethod = "GET" + rctx.RoutePath = routePath + next.ServeHTTP(w, r) + return + } + } + + next.ServeHTTP(w, r) + }) +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/heartbeat.go b/vendor/github.com/go-chi/chi/v5/middleware/heartbeat.go new file mode 100644 index 00000000..f36e8ccf --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/heartbeat.go @@ -0,0 +1,26 @@ +package middleware + +import ( + "net/http" + "strings" +) + +// Heartbeat endpoint middleware useful to setting up a path like +// `/ping` that load balancers or uptime testing external services +// can make a request before hitting any routes. It's also convenient +// to place this above ACL middlewares as well. +func Heartbeat(endpoint string) func(http.Handler) http.Handler { + f := func(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + if (r.Method == "GET" || r.Method == "HEAD") && strings.EqualFold(r.URL.Path, endpoint) { + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + w.Write([]byte(".")) + return + } + h.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) + } + return f +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/logger.go b/vendor/github.com/go-chi/chi/v5/middleware/logger.go new file mode 100644 index 00000000..98250d82 --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/logger.go @@ -0,0 +1,171 @@ +package middleware + +import ( + "bytes" + "context" + "log" + "net/http" + "os" + "runtime" + "time" +) + +var ( + // LogEntryCtxKey is the context.Context key to store the request log entry. + LogEntryCtxKey = &contextKey{"LogEntry"} + + // DefaultLogger is called by the Logger middleware handler to log each request. + // Its made a package-level variable so that it can be reconfigured for custom + // logging configurations. + DefaultLogger func(next http.Handler) http.Handler +) + +// Logger is a middleware that logs the start and end of each request, along +// with some useful data about what was requested, what the response status was, +// and how long it took to return. When standard output is a TTY, Logger will +// print in color, otherwise it will print in black and white. Logger prints a +// request ID if one is provided. +// +// Alternatively, look at https://github.com/goware/httplog for a more in-depth +// http logger with structured logging support. +// +// IMPORTANT NOTE: Logger should go before any other middleware that may change +// the response, such as middleware.Recoverer. Example: +// r := chi.NewRouter() +// r.Use(middleware.Logger) // <--<< Logger should come before Recoverer +// r.Use(middleware.Recoverer) +// r.Get("/", handler) +func Logger(next http.Handler) http.Handler { + return DefaultLogger(next) +} + +// RequestLogger returns a logger handler using a custom LogFormatter. +func RequestLogger(f LogFormatter) func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + entry := f.NewLogEntry(r) + ww := NewWrapResponseWriter(w, r.ProtoMajor) + + t1 := time.Now() + defer func() { + entry.Write(ww.Status(), ww.BytesWritten(), ww.Header(), time.Since(t1), nil) + }() + + next.ServeHTTP(ww, WithLogEntry(r, entry)) + } + return http.HandlerFunc(fn) + } +} + +// LogFormatter initiates the beginning of a new LogEntry per request. +// See DefaultLogFormatter for an example implementation. +type LogFormatter interface { + NewLogEntry(r *http.Request) LogEntry +} + +// LogEntry records the final log when a request completes. +// See defaultLogEntry for an example implementation. +type LogEntry interface { + Write(status, bytes int, header http.Header, elapsed time.Duration, extra interface{}) + Panic(v interface{}, stack []byte) +} + +// GetLogEntry returns the in-context LogEntry for a request. +func GetLogEntry(r *http.Request) LogEntry { + entry, _ := r.Context().Value(LogEntryCtxKey).(LogEntry) + return entry +} + +// WithLogEntry sets the in-context LogEntry for a request. +func WithLogEntry(r *http.Request, entry LogEntry) *http.Request { + r = r.WithContext(context.WithValue(r.Context(), LogEntryCtxKey, entry)) + return r +} + +// LoggerInterface accepts printing to stdlib logger or compatible logger. +type LoggerInterface interface { + Print(v ...interface{}) +} + +// DefaultLogFormatter is a simple logger that implements a LogFormatter. +type DefaultLogFormatter struct { + Logger LoggerInterface + NoColor bool +} + +// NewLogEntry creates a new LogEntry for the request. +func (l *DefaultLogFormatter) NewLogEntry(r *http.Request) LogEntry { + useColor := !l.NoColor + entry := &defaultLogEntry{ + DefaultLogFormatter: l, + request: r, + buf: &bytes.Buffer{}, + useColor: useColor, + } + + reqID := GetReqID(r.Context()) + if reqID != "" { + cW(entry.buf, useColor, nYellow, "[%s] ", reqID) + } + cW(entry.buf, useColor, nCyan, "\"") + cW(entry.buf, useColor, bMagenta, "%s ", r.Method) + + scheme := "http" + if r.TLS != nil { + scheme = "https" + } + cW(entry.buf, useColor, nCyan, "%s://%s%s %s\" ", scheme, r.Host, r.RequestURI, r.Proto) + + entry.buf.WriteString("from ") + entry.buf.WriteString(r.RemoteAddr) + entry.buf.WriteString(" - ") + + return entry +} + +type defaultLogEntry struct { + *DefaultLogFormatter + request *http.Request + buf *bytes.Buffer + useColor bool +} + +func (l *defaultLogEntry) Write(status, bytes int, header http.Header, elapsed time.Duration, extra interface{}) { + switch { + case status < 200: + cW(l.buf, l.useColor, bBlue, "%03d", status) + case status < 300: + cW(l.buf, l.useColor, bGreen, "%03d", status) + case status < 400: + cW(l.buf, l.useColor, bCyan, "%03d", status) + case status < 500: + cW(l.buf, l.useColor, bYellow, "%03d", status) + default: + cW(l.buf, l.useColor, bRed, "%03d", status) + } + + cW(l.buf, l.useColor, bBlue, " %dB", bytes) + + l.buf.WriteString(" in ") + if elapsed < 500*time.Millisecond { + cW(l.buf, l.useColor, nGreen, "%s", elapsed) + } else if elapsed < 5*time.Second { + cW(l.buf, l.useColor, nYellow, "%s", elapsed) + } else { + cW(l.buf, l.useColor, nRed, "%s", elapsed) + } + + l.Logger.Print(l.buf.String()) +} + +func (l *defaultLogEntry) Panic(v interface{}, stack []byte) { + PrintPrettyStack(v) +} + +func init() { + color := true + if runtime.GOOS == "windows" { + color = false + } + DefaultLogger = RequestLogger(&DefaultLogFormatter{Logger: log.New(os.Stdout, "", log.LstdFlags), NoColor: !color}) +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/maybe.go b/vendor/github.com/go-chi/chi/v5/middleware/maybe.go new file mode 100644 index 00000000..d8ca63b9 --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/maybe.go @@ -0,0 +1,18 @@ +package middleware + +import "net/http" + +// Maybe middleware will allow you to change the flow of the middleware stack execution depending on return +// value of maybeFn(request). This is useful for example if you'd like to skip a middleware handler if +// a request does not satisfied the maybeFn logic. +func Maybe(mw func(http.Handler) http.Handler, maybeFn func(r *http.Request) bool) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if maybeFn(r) { + mw(next).ServeHTTP(w, r) + } else { + next.ServeHTTP(w, r) + } + }) + } +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/middleware.go b/vendor/github.com/go-chi/chi/v5/middleware/middleware.go new file mode 100644 index 00000000..cc371e00 --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/middleware.go @@ -0,0 +1,23 @@ +package middleware + +import "net/http" + +// New will create a new middleware handler from a http.Handler. +func New(h http.Handler) func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h.ServeHTTP(w, r) + }) + } +} + +// contextKey is a value for use with context.WithValue. It's used as +// a pointer so it fits in an interface{} without allocation. This technique +// for defining context keys was copied from Go 1.7's new use of context in net/http. +type contextKey struct { + name string +} + +func (k *contextKey) String() string { + return "chi/middleware context value " + k.name +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/nocache.go b/vendor/github.com/go-chi/chi/v5/middleware/nocache.go new file mode 100644 index 00000000..7353448d --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/nocache.go @@ -0,0 +1,58 @@ +package middleware + +// Ported from Goji's middleware, source: +// https://github.com/zenazn/goji/tree/master/web/middleware + +import ( + "net/http" + "time" +) + +// Unix epoch time +var epoch = time.Unix(0, 0).UTC().Format(http.TimeFormat) + +// Taken from https://github.com/mytrile/nocache +var noCacheHeaders = map[string]string{ + "Expires": epoch, + "Cache-Control": "no-cache, no-store, no-transform, must-revalidate, private, max-age=0", + "Pragma": "no-cache", + "X-Accel-Expires": "0", +} + +var etagHeaders = []string{ + "ETag", + "If-Modified-Since", + "If-Match", + "If-None-Match", + "If-Range", + "If-Unmodified-Since", +} + +// NoCache is a simple piece of middleware that sets a number of HTTP headers to prevent +// a router (or subrouter) from being cached by an upstream proxy and/or client. +// +// As per http://wiki.nginx.org/HttpProxyModule - NoCache sets: +// Expires: Thu, 01 Jan 1970 00:00:00 UTC +// Cache-Control: no-cache, private, max-age=0 +// X-Accel-Expires: 0 +// Pragma: no-cache (for HTTP/1.0 proxies/clients) +func NoCache(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + + // Delete any ETag headers that may have been set + for _, v := range etagHeaders { + if r.Header.Get(v) != "" { + r.Header.Del(v) + } + } + + // Set our NoCache headers + for k, v := range noCacheHeaders { + w.Header().Set(k, v) + } + + h.ServeHTTP(w, r) + } + + return http.HandlerFunc(fn) +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/page_route.go b/vendor/github.com/go-chi/chi/v5/middleware/page_route.go new file mode 100644 index 00000000..32871b7e --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/page_route.go @@ -0,0 +1,20 @@ +package middleware + +import ( + "net/http" + "strings" +) + +// PageRoute is a simple middleware which allows you to route a static GET request +// at the middleware stack level. +func PageRoute(path string, handler http.Handler) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "GET" && strings.EqualFold(r.URL.Path, path) { + handler.ServeHTTP(w, r) + return + } + next.ServeHTTP(w, r) + }) + } +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/path_rewrite.go b/vendor/github.com/go-chi/chi/v5/middleware/path_rewrite.go new file mode 100644 index 00000000..99af62c0 --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/path_rewrite.go @@ -0,0 +1,16 @@ +package middleware + +import ( + "net/http" + "strings" +) + +// PathRewrite is a simple middleware which allows you to rewrite the request URL path. +func PathRewrite(old, new string) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.URL.Path = strings.Replace(r.URL.Path, old, new, 1) + next.ServeHTTP(w, r) + }) + } +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/profiler.go b/vendor/github.com/go-chi/chi/v5/middleware/profiler.go new file mode 100644 index 00000000..3c36f878 --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/profiler.go @@ -0,0 +1,62 @@ +package middleware + +import ( + "expvar" + "fmt" + "net/http" + "net/http/pprof" + + "github.com/go-chi/chi/v5" +) + +// Profiler is a convenient subrouter used for mounting net/http/pprof. ie. +// +// func MyService() http.Handler { +// r := chi.NewRouter() +// // ..middlewares +// r.Mount("/debug", middleware.Profiler()) +// // ..routes +// return r +// } +func Profiler() http.Handler { + r := chi.NewRouter() + r.Use(NoCache) + + r.Get("/", func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, r.RequestURI+"/pprof/", http.StatusMovedPermanently) + }) + r.HandleFunc("/pprof", func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, r.RequestURI+"/", http.StatusMovedPermanently) + }) + + r.HandleFunc("/pprof/*", pprof.Index) + r.HandleFunc("/pprof/cmdline", pprof.Cmdline) + r.HandleFunc("/pprof/profile", pprof.Profile) + r.HandleFunc("/pprof/symbol", pprof.Symbol) + r.HandleFunc("/pprof/trace", pprof.Trace) + r.HandleFunc("/vars", expVars) + + r.Handle("/pprof/goroutine", pprof.Handler("goroutine")) + r.Handle("/pprof/threadcreate", pprof.Handler("threadcreate")) + r.Handle("/pprof/mutex", pprof.Handler("mutex")) + r.Handle("/pprof/heap", pprof.Handler("heap")) + r.Handle("/pprof/block", pprof.Handler("block")) + r.Handle("/pprof/allocs", pprof.Handler("allocs")) + + return r +} + +// Replicated from expvar.go as not public. +func expVars(w http.ResponseWriter, r *http.Request) { + first := true + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, "{\n") + expvar.Do(func(kv expvar.KeyValue) { + if !first { + fmt.Fprintf(w, ",\n") + } + first = false + fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value) + }) + fmt.Fprintf(w, "\n}\n") +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/realip.go b/vendor/github.com/go-chi/chi/v5/middleware/realip.go new file mode 100644 index 00000000..2c6b3b33 --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/realip.go @@ -0,0 +1,60 @@ +package middleware + +// Ported from Goji's middleware, source: +// https://github.com/zenazn/goji/tree/master/web/middleware + +import ( + "net" + "net/http" + "strings" +) + +var trueClientIP = http.CanonicalHeaderKey("True-Client-IP") +var xForwardedFor = http.CanonicalHeaderKey("X-Forwarded-For") +var xRealIP = http.CanonicalHeaderKey("X-Real-IP") + +// RealIP is a middleware that sets a http.Request's RemoteAddr to the results +// of parsing either the True-Client-IP, X-Real-IP or the X-Forwarded-For headers +// (in that order). +// +// This middleware should be inserted fairly early in the middleware stack to +// ensure that subsequent layers (e.g., request loggers) which examine the +// RemoteAddr will see the intended value. +// +// You should only use this middleware if you can trust the headers passed to +// you (in particular, the two headers this middleware uses), for example +// because you have placed a reverse proxy like HAProxy or nginx in front of +// chi. If your reverse proxies are configured to pass along arbitrary header +// values from the client, or if you use this middleware without a reverse +// proxy, malicious clients will be able to make you very sad (or, depending on +// how you're using RemoteAddr, vulnerable to an attack of some sort). +func RealIP(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + if rip := realIP(r); rip != "" { + r.RemoteAddr = rip + } + h.ServeHTTP(w, r) + } + + return http.HandlerFunc(fn) +} + +func realIP(r *http.Request) string { + var ip string + + if tcip := r.Header.Get(trueClientIP); tcip != "" { + ip = tcip + } else if xrip := r.Header.Get(xRealIP); xrip != "" { + ip = xrip + } else if xff := r.Header.Get(xForwardedFor); xff != "" { + i := strings.Index(xff, ",") + if i == -1 { + i = len(xff) + } + ip = xff[:i] + } + if ip == "" || net.ParseIP(ip) == nil { + return "" + } + return ip +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/recoverer.go b/vendor/github.com/go-chi/chi/v5/middleware/recoverer.go new file mode 100644 index 00000000..612e155a --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/recoverer.go @@ -0,0 +1,204 @@ +package middleware + +// The original work was derived from Goji's middleware, source: +// https://github.com/zenazn/goji/tree/master/web/middleware + +import ( + "bytes" + "errors" + "fmt" + "io" + "net/http" + "os" + "runtime/debug" + "strings" +) + +// Recoverer is a middleware that recovers from panics, logs the panic (and a +// backtrace), and returns a HTTP 500 (Internal Server Error) status if +// possible. Recoverer prints a request ID if one is provided. +// +// Alternatively, look at https://github.com/go-chi/httplog middleware pkgs. +func Recoverer(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + defer func() { + if rvr := recover(); rvr != nil { + if rvr == http.ErrAbortHandler { + // we don't recover http.ErrAbortHandler so the response + // to the client is aborted, this should not be logged + panic(rvr) + } + + logEntry := GetLogEntry(r) + if logEntry != nil { + logEntry.Panic(rvr, debug.Stack()) + } else { + PrintPrettyStack(rvr) + } + + if r.Header.Get("Connection") != "Upgrade" { + w.WriteHeader(http.StatusInternalServerError) + } + } + }() + + next.ServeHTTP(w, r) + } + + return http.HandlerFunc(fn) +} + +// for ability to test the PrintPrettyStack function +var recovererErrorWriter io.Writer = os.Stderr + +func PrintPrettyStack(rvr interface{}) { + debugStack := debug.Stack() + s := prettyStack{} + out, err := s.parse(debugStack, rvr) + if err == nil { + recovererErrorWriter.Write(out) + } else { + // print stdlib output as a fallback + os.Stderr.Write(debugStack) + } +} + +type prettyStack struct { +} + +func (s prettyStack) parse(debugStack []byte, rvr interface{}) ([]byte, error) { + var err error + useColor := true + buf := &bytes.Buffer{} + + cW(buf, false, bRed, "\n") + cW(buf, useColor, bCyan, " panic: ") + cW(buf, useColor, bBlue, "%v", rvr) + cW(buf, false, bWhite, "\n \n") + + // process debug stack info + stack := strings.Split(string(debugStack), "\n") + lines := []string{} + + // locate panic line, as we may have nested panics + for i := len(stack) - 1; i > 0; i-- { + lines = append(lines, stack[i]) + if strings.HasPrefix(stack[i], "panic(") { + lines = lines[0 : len(lines)-2] // remove boilerplate + break + } + } + + // reverse + for i := len(lines)/2 - 1; i >= 0; i-- { + opp := len(lines) - 1 - i + lines[i], lines[opp] = lines[opp], lines[i] + } + + // decorate + for i, line := range lines { + lines[i], err = s.decorateLine(line, useColor, i) + if err != nil { + return nil, err + } + } + + for _, l := range lines { + fmt.Fprintf(buf, "%s", l) + } + return buf.Bytes(), nil +} + +func (s prettyStack) decorateLine(line string, useColor bool, num int) (string, error) { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "\t") || strings.Contains(line, ".go:") { + return s.decorateSourceLine(line, useColor, num) + } else if strings.HasSuffix(line, ")") { + return s.decorateFuncCallLine(line, useColor, num) + } else { + if strings.HasPrefix(line, "\t") { + return strings.Replace(line, "\t", " ", 1), nil + } else { + return fmt.Sprintf(" %s\n", line), nil + } + } +} + +func (s prettyStack) decorateFuncCallLine(line string, useColor bool, num int) (string, error) { + idx := strings.LastIndex(line, "(") + if idx < 0 { + return "", errors.New("not a func call line") + } + + buf := &bytes.Buffer{} + pkg := line[0:idx] + // addr := line[idx:] + method := "" + + if idx := strings.LastIndex(pkg, string(os.PathSeparator)); idx < 0 { + if idx := strings.Index(pkg, "."); idx > 0 { + method = pkg[idx:] + pkg = pkg[0:idx] + } + } else { + method = pkg[idx+1:] + pkg = pkg[0 : idx+1] + if idx := strings.Index(method, "."); idx > 0 { + pkg += method[0:idx] + method = method[idx:] + } + } + pkgColor := nYellow + methodColor := bGreen + + if num == 0 { + cW(buf, useColor, bRed, " -> ") + pkgColor = bMagenta + methodColor = bRed + } else { + cW(buf, useColor, bWhite, " ") + } + cW(buf, useColor, pkgColor, "%s", pkg) + cW(buf, useColor, methodColor, "%s\n", method) + // cW(buf, useColor, nBlack, "%s", addr) + return buf.String(), nil +} + +func (s prettyStack) decorateSourceLine(line string, useColor bool, num int) (string, error) { + idx := strings.LastIndex(line, ".go:") + if idx < 0 { + return "", errors.New("not a source line") + } + + buf := &bytes.Buffer{} + path := line[0 : idx+3] + lineno := line[idx+3:] + + idx = strings.LastIndex(path, string(os.PathSeparator)) + dir := path[0 : idx+1] + file := path[idx+1:] + + idx = strings.Index(lineno, " ") + if idx > 0 { + lineno = lineno[0:idx] + } + fileColor := bCyan + lineColor := bGreen + + if num == 1 { + cW(buf, useColor, bRed, " -> ") + fileColor = bRed + lineColor = bMagenta + } else { + cW(buf, false, bWhite, " ") + } + cW(buf, useColor, bWhite, "%s", dir) + cW(buf, useColor, fileColor, "%s", file) + cW(buf, useColor, lineColor, "%s", lineno) + if num == 1 { + cW(buf, false, bWhite, "\n") + } + cW(buf, false, bWhite, "\n") + + return buf.String(), nil +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/request_id.go b/vendor/github.com/go-chi/chi/v5/middleware/request_id.go new file mode 100644 index 00000000..4903ecc2 --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/request_id.go @@ -0,0 +1,96 @@ +package middleware + +// Ported from Goji's middleware, source: +// https://github.com/zenazn/goji/tree/master/web/middleware + +import ( + "context" + "crypto/rand" + "encoding/base64" + "fmt" + "net/http" + "os" + "strings" + "sync/atomic" +) + +// Key to use when setting the request ID. +type ctxKeyRequestID int + +// RequestIDKey is the key that holds the unique request ID in a request context. +const RequestIDKey ctxKeyRequestID = 0 + +// RequestIDHeader is the name of the HTTP Header which contains the request id. +// Exported so that it can be changed by developers +var RequestIDHeader = "X-Request-Id" + +var prefix string +var reqid uint64 + +// A quick note on the statistics here: we're trying to calculate the chance that +// two randomly generated base62 prefixes will collide. We use the formula from +// http://en.wikipedia.org/wiki/Birthday_problem +// +// P[m, n] \approx 1 - e^{-m^2/2n} +// +// We ballpark an upper bound for $m$ by imagining (for whatever reason) a server +// that restarts every second over 10 years, for $m = 86400 * 365 * 10 = 315360000$ +// +// For a $k$ character base-62 identifier, we have $n(k) = 62^k$ +// +// Plugging this in, we find $P[m, n(10)] \approx 5.75%$, which is good enough for +// our purposes, and is surely more than anyone would ever need in practice -- a +// process that is rebooted a handful of times a day for a hundred years has less +// than a millionth of a percent chance of generating two colliding IDs. + +func init() { + hostname, err := os.Hostname() + if hostname == "" || err != nil { + hostname = "localhost" + } + var buf [12]byte + var b64 string + for len(b64) < 10 { + rand.Read(buf[:]) + b64 = base64.StdEncoding.EncodeToString(buf[:]) + b64 = strings.NewReplacer("+", "", "/", "").Replace(b64) + } + + prefix = fmt.Sprintf("%s/%s", hostname, b64[0:10]) +} + +// RequestID is a middleware that injects a request ID into the context of each +// request. A request ID is a string of the form "host.example.com/random-0001", +// where "random" is a base62 random string that uniquely identifies this go +// process, and where the last number is an atomically incremented request +// counter. +func RequestID(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + requestID := r.Header.Get(RequestIDHeader) + if requestID == "" { + myid := atomic.AddUint64(&reqid, 1) + requestID = fmt.Sprintf("%s-%06d", prefix, myid) + } + ctx = context.WithValue(ctx, RequestIDKey, requestID) + next.ServeHTTP(w, r.WithContext(ctx)) + } + return http.HandlerFunc(fn) +} + +// GetReqID returns a request ID from the given context if one is present. +// Returns the empty string if a request ID cannot be found. +func GetReqID(ctx context.Context) string { + if ctx == nil { + return "" + } + if reqID, ok := ctx.Value(RequestIDKey).(string); ok { + return reqID + } + return "" +} + +// NextRequestID generates the next request ID in the sequence. +func NextRequestID() uint64 { + return atomic.AddUint64(&reqid, 1) +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/request_size.go b/vendor/github.com/go-chi/chi/v5/middleware/request_size.go new file mode 100644 index 00000000..678248c4 --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/request_size.go @@ -0,0 +1,18 @@ +package middleware + +import ( + "net/http" +) + +// RequestSize is a middleware that will limit request sizes to a specified +// number of bytes. It uses MaxBytesReader to do so. +func RequestSize(bytes int64) func(http.Handler) http.Handler { + f := func(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, bytes) + h.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) + } + return f +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/route_headers.go b/vendor/github.com/go-chi/chi/v5/middleware/route_headers.go new file mode 100644 index 00000000..ea914a1d --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/route_headers.go @@ -0,0 +1,160 @@ +package middleware + +import ( + "net/http" + "strings" +) + +// RouteHeaders is a neat little header-based router that allows you to direct +// the flow of a request through a middleware stack based on a request header. +// +// For example, lets say you'd like to setup multiple routers depending on the +// request Host header, you could then do something as so: +// +// r := chi.NewRouter() +// rSubdomain := chi.NewRouter() +// +// r.Use(middleware.RouteHeaders(). +// Route("Host", "example.com", middleware.New(r)). +// Route("Host", "*.example.com", middleware.New(rSubdomain)). +// Handler) +// +// r.Get("/", h) +// rSubdomain.Get("/", h2) +// +// +// Another example, imagine you want to setup multiple CORS handlers, where for +// your origin servers you allow authorized requests, but for third-party public +// requests, authorization is disabled. +// +// r := chi.NewRouter() +// +// r.Use(middleware.RouteHeaders(). +// Route("Origin", "https://app.skyweaver.net", cors.Handler(cors.Options{ +// AllowedOrigins: []string{"https://api.skyweaver.net"}, +// AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}, +// AllowedHeaders: []string{"Accept", "Authorization", "Content-Type"}, +// AllowCredentials: true, // <----------<<< allow credentials +// })). +// Route("Origin", "*", cors.Handler(cors.Options{ +// AllowedOrigins: []string{"*"}, +// AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}, +// AllowedHeaders: []string{"Accept", "Content-Type"}, +// AllowCredentials: false, // <----------<<< do not allow credentials +// })). +// Handler) +// +func RouteHeaders() HeaderRouter { + return HeaderRouter{} +} + +type HeaderRouter map[string][]HeaderRoute + +func (hr HeaderRouter) Route(header, match string, middlewareHandler func(next http.Handler) http.Handler) HeaderRouter { + header = strings.ToLower(header) + k := hr[header] + if k == nil { + hr[header] = []HeaderRoute{} + } + hr[header] = append(hr[header], HeaderRoute{MatchOne: NewPattern(match), Middleware: middlewareHandler}) + return hr +} + +func (hr HeaderRouter) RouteAny(header string, match []string, middlewareHandler func(next http.Handler) http.Handler) HeaderRouter { + header = strings.ToLower(header) + k := hr[header] + if k == nil { + hr[header] = []HeaderRoute{} + } + patterns := []Pattern{} + for _, m := range match { + patterns = append(patterns, NewPattern(m)) + } + hr[header] = append(hr[header], HeaderRoute{MatchAny: patterns, Middleware: middlewareHandler}) + return hr +} + +func (hr HeaderRouter) RouteDefault(handler func(next http.Handler) http.Handler) HeaderRouter { + hr["*"] = []HeaderRoute{{Middleware: handler}} + return hr +} + +func (hr HeaderRouter) Handler(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if len(hr) == 0 { + // skip if no routes set + next.ServeHTTP(w, r) + } + + // find first matching header route, and continue + for header, matchers := range hr { + headerValue := r.Header.Get(header) + if headerValue == "" { + continue + } + headerValue = strings.ToLower(headerValue) + for _, matcher := range matchers { + if matcher.IsMatch(headerValue) { + matcher.Middleware(next).ServeHTTP(w, r) + return + } + } + } + + // if no match, check for "*" default route + matcher, ok := hr["*"] + if !ok || matcher[0].Middleware == nil { + next.ServeHTTP(w, r) + return + } + matcher[0].Middleware(next).ServeHTTP(w, r) + }) +} + +type HeaderRoute struct { + Middleware func(next http.Handler) http.Handler + MatchOne Pattern + MatchAny []Pattern +} + +func (r HeaderRoute) IsMatch(value string) bool { + if len(r.MatchAny) > 0 { + for _, m := range r.MatchAny { + if m.Match(value) { + return true + } + } + } else if r.MatchOne.Match(value) { + return true + } + return false +} + +type Pattern struct { + prefix string + suffix string + wildcard bool +} + +func NewPattern(value string) Pattern { + p := Pattern{} + if i := strings.IndexByte(value, '*'); i >= 0 { + p.wildcard = true + p.prefix = value[0:i] + p.suffix = value[i+1:] + } else { + p.prefix = value + } + return p +} + +func (p Pattern) Match(v string) bool { + if !p.wildcard { + if p.prefix == v { + return true + } else { + return false + } + } + return len(v) >= len(p.prefix+p.suffix) && strings.HasPrefix(v, p.prefix) && strings.HasSuffix(v, p.suffix) +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/strip.go b/vendor/github.com/go-chi/chi/v5/middleware/strip.go new file mode 100644 index 00000000..ce8ebfcc --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/strip.go @@ -0,0 +1,62 @@ +package middleware + +import ( + "fmt" + "net/http" + + "github.com/go-chi/chi/v5" +) + +// StripSlashes is a middleware that will match request paths with a trailing +// slash, strip it from the path and continue routing through the mux, if a route +// matches, then it will serve the handler. +func StripSlashes(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + var path string + rctx := chi.RouteContext(r.Context()) + if rctx != nil && rctx.RoutePath != "" { + path = rctx.RoutePath + } else { + path = r.URL.Path + } + if len(path) > 1 && path[len(path)-1] == '/' { + newPath := path[:len(path)-1] + if rctx == nil { + r.URL.Path = newPath + } else { + rctx.RoutePath = newPath + } + } + next.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) +} + +// RedirectSlashes is a middleware that will match request paths with a trailing +// slash and redirect to the same path, less the trailing slash. +// +// NOTE: RedirectSlashes middleware is *incompatible* with http.FileServer, +// see https://github.com/go-chi/chi/issues/343 +func RedirectSlashes(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + var path string + rctx := chi.RouteContext(r.Context()) + if rctx != nil && rctx.RoutePath != "" { + path = rctx.RoutePath + } else { + path = r.URL.Path + } + if len(path) > 1 && path[len(path)-1] == '/' { + if r.URL.RawQuery != "" { + path = fmt.Sprintf("%s?%s", path[:len(path)-1], r.URL.RawQuery) + } else { + path = path[:len(path)-1] + } + redirectURL := fmt.Sprintf("//%s%s", r.Host, path) + http.Redirect(w, r, redirectURL, 301) + return + } + next.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/terminal.go b/vendor/github.com/go-chi/chi/v5/middleware/terminal.go new file mode 100644 index 00000000..5ead7b92 --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/terminal.go @@ -0,0 +1,63 @@ +package middleware + +// Ported from Goji's middleware, source: +// https://github.com/zenazn/goji/tree/master/web/middleware + +import ( + "fmt" + "io" + "os" +) + +var ( + // Normal colors + nBlack = []byte{'\033', '[', '3', '0', 'm'} + nRed = []byte{'\033', '[', '3', '1', 'm'} + nGreen = []byte{'\033', '[', '3', '2', 'm'} + nYellow = []byte{'\033', '[', '3', '3', 'm'} + nBlue = []byte{'\033', '[', '3', '4', 'm'} + nMagenta = []byte{'\033', '[', '3', '5', 'm'} + nCyan = []byte{'\033', '[', '3', '6', 'm'} + nWhite = []byte{'\033', '[', '3', '7', 'm'} + // Bright colors + bBlack = []byte{'\033', '[', '3', '0', ';', '1', 'm'} + bRed = []byte{'\033', '[', '3', '1', ';', '1', 'm'} + bGreen = []byte{'\033', '[', '3', '2', ';', '1', 'm'} + bYellow = []byte{'\033', '[', '3', '3', ';', '1', 'm'} + bBlue = []byte{'\033', '[', '3', '4', ';', '1', 'm'} + bMagenta = []byte{'\033', '[', '3', '5', ';', '1', 'm'} + bCyan = []byte{'\033', '[', '3', '6', ';', '1', 'm'} + bWhite = []byte{'\033', '[', '3', '7', ';', '1', 'm'} + + reset = []byte{'\033', '[', '0', 'm'} +) + +var IsTTY bool + +func init() { + // This is sort of cheating: if stdout is a character device, we assume + // that means it's a TTY. Unfortunately, there are many non-TTY + // character devices, but fortunately stdout is rarely set to any of + // them. + // + // We could solve this properly by pulling in a dependency on + // code.google.com/p/go.crypto/ssh/terminal, for instance, but as a + // heuristic for whether to print in color or in black-and-white, I'd + // really rather not. + fi, err := os.Stdout.Stat() + if err == nil { + m := os.ModeDevice | os.ModeCharDevice + IsTTY = fi.Mode()&m == m + } +} + +// colorWrite +func cW(w io.Writer, useColor bool, color []byte, s string, args ...interface{}) { + if IsTTY && useColor { + w.Write(color) + } + fmt.Fprintf(w, s, args...) + if IsTTY && useColor { + w.Write(reset) + } +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/throttle.go b/vendor/github.com/go-chi/chi/v5/middleware/throttle.go new file mode 100644 index 00000000..bdf4f9f1 --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/throttle.go @@ -0,0 +1,132 @@ +package middleware + +import ( + "net/http" + "strconv" + "time" +) + +const ( + errCapacityExceeded = "Server capacity exceeded." + errTimedOut = "Timed out while waiting for a pending request to complete." + errContextCanceled = "Context was canceled." +) + +var ( + defaultBacklogTimeout = time.Second * 60 +) + +// ThrottleOpts represents a set of throttling options. +type ThrottleOpts struct { + RetryAfterFn func(ctxDone bool) time.Duration + Limit int + BacklogLimit int + BacklogTimeout time.Duration +} + +// Throttle is a middleware that limits number of currently processed requests +// at a time across all users. Note: Throttle is not a rate-limiter per user, +// instead it just puts a ceiling on the number of currently in-flight requests +// being processed from the point from where the Throttle middleware is mounted. +func Throttle(limit int) func(http.Handler) http.Handler { + return ThrottleWithOpts(ThrottleOpts{Limit: limit, BacklogTimeout: defaultBacklogTimeout}) +} + +// ThrottleBacklog is a middleware that limits number of currently processed +// requests at a time and provides a backlog for holding a finite number of +// pending requests. +func ThrottleBacklog(limit, backlogLimit int, backlogTimeout time.Duration) func(http.Handler) http.Handler { + return ThrottleWithOpts(ThrottleOpts{Limit: limit, BacklogLimit: backlogLimit, BacklogTimeout: backlogTimeout}) +} + +// ThrottleWithOpts is a middleware that limits number of currently processed requests using passed ThrottleOpts. +func ThrottleWithOpts(opts ThrottleOpts) func(http.Handler) http.Handler { + if opts.Limit < 1 { + panic("chi/middleware: Throttle expects limit > 0") + } + + if opts.BacklogLimit < 0 { + panic("chi/middleware: Throttle expects backlogLimit to be positive") + } + + t := throttler{ + tokens: make(chan token, opts.Limit), + backlogTokens: make(chan token, opts.Limit+opts.BacklogLimit), + backlogTimeout: opts.BacklogTimeout, + retryAfterFn: opts.RetryAfterFn, + } + + // Filling tokens. + for i := 0; i < opts.Limit+opts.BacklogLimit; i++ { + if i < opts.Limit { + t.tokens <- token{} + } + t.backlogTokens <- token{} + } + + return func(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + select { + + case <-ctx.Done(): + t.setRetryAfterHeaderIfNeeded(w, true) + http.Error(w, errContextCanceled, http.StatusTooManyRequests) + return + + case btok := <-t.backlogTokens: + timer := time.NewTimer(t.backlogTimeout) + + defer func() { + t.backlogTokens <- btok + }() + + select { + case <-timer.C: + t.setRetryAfterHeaderIfNeeded(w, false) + http.Error(w, errTimedOut, http.StatusTooManyRequests) + return + case <-ctx.Done(): + timer.Stop() + t.setRetryAfterHeaderIfNeeded(w, true) + http.Error(w, errContextCanceled, http.StatusTooManyRequests) + return + case tok := <-t.tokens: + defer func() { + timer.Stop() + t.tokens <- tok + }() + next.ServeHTTP(w, r) + } + return + + default: + t.setRetryAfterHeaderIfNeeded(w, false) + http.Error(w, errCapacityExceeded, http.StatusTooManyRequests) + return + } + } + + return http.HandlerFunc(fn) + } +} + +// token represents a request that is being processed. +type token struct{} + +// throttler limits number of currently processed requests at a time. +type throttler struct { + tokens chan token + backlogTokens chan token + retryAfterFn func(ctxDone bool) time.Duration + backlogTimeout time.Duration +} + +// setRetryAfterHeaderIfNeeded sets Retry-After HTTP header if corresponding retryAfterFn option of throttler is initialized. +func (t throttler) setRetryAfterHeaderIfNeeded(w http.ResponseWriter, ctxDone bool) { + if t.retryAfterFn == nil { + return + } + w.Header().Set("Retry-After", strconv.Itoa(int(t.retryAfterFn(ctxDone).Seconds()))) +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/timeout.go b/vendor/github.com/go-chi/chi/v5/middleware/timeout.go new file mode 100644 index 00000000..8e373536 --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/timeout.go @@ -0,0 +1,49 @@ +package middleware + +import ( + "context" + "net/http" + "time" +) + +// Timeout is a middleware that cancels ctx after a given timeout and return +// a 504 Gateway Timeout error to the client. +// +// It's required that you select the ctx.Done() channel to check for the signal +// if the context has reached its deadline and return, otherwise the timeout +// signal will be just ignored. +// +// ie. a route/handler may look like: +// +// r.Get("/long", func(w http.ResponseWriter, r *http.Request) { +// ctx := r.Context() +// processTime := time.Duration(rand.Intn(4)+1) * time.Second +// +// select { +// case <-ctx.Done(): +// return +// +// case <-time.After(processTime): +// // The above channel simulates some hard work. +// } +// +// w.Write([]byte("done")) +// }) +// +func Timeout(timeout time.Duration) func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), timeout) + defer func() { + cancel() + if ctx.Err() == context.DeadlineExceeded { + w.WriteHeader(http.StatusGatewayTimeout) + } + }() + + r = r.WithContext(ctx) + next.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) + } +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/url_format.go b/vendor/github.com/go-chi/chi/v5/middleware/url_format.go new file mode 100644 index 00000000..919eb0fe --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/url_format.go @@ -0,0 +1,76 @@ +package middleware + +import ( + "context" + "net/http" + "strings" + + "github.com/go-chi/chi/v5" +) + +var ( + // URLFormatCtxKey is the context.Context key to store the URL format data + // for a request. + URLFormatCtxKey = &contextKey{"URLFormat"} +) + +// URLFormat is a middleware that parses the url extension from a request path and stores it +// on the context as a string under the key `middleware.URLFormatCtxKey`. The middleware will +// trim the suffix from the routing path and continue routing. +// +// Routers should not include a url parameter for the suffix when using this middleware. +// +// Sample usage.. for url paths: `/articles/1`, `/articles/1.json` and `/articles/1.xml` +// +// func routes() http.Handler { +// r := chi.NewRouter() +// r.Use(middleware.URLFormat) +// +// r.Get("/articles/{id}", ListArticles) +// +// return r +// } +// +// func ListArticles(w http.ResponseWriter, r *http.Request) { +// urlFormat, _ := r.Context().Value(middleware.URLFormatCtxKey).(string) +// +// switch urlFormat { +// case "json": +// render.JSON(w, r, articles) +// case "xml:" +// render.XML(w, r, articles) +// default: +// render.JSON(w, r, articles) +// } +// } +// +func URLFormat(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + var format string + path := r.URL.Path + + rctx := chi.RouteContext(r.Context()) + if rctx != nil && rctx.RoutePath != "" { + path = rctx.RoutePath + } + + if strings.Index(path, ".") > 0 { + base := strings.LastIndex(path, "/") + idx := strings.LastIndex(path[base:], ".") + + if idx > 0 { + idx += base + format = path[idx+1:] + + rctx.RoutePath = path[:idx] + } + } + + r = r.WithContext(context.WithValue(ctx, URLFormatCtxKey, format)) + + next.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/value.go b/vendor/github.com/go-chi/chi/v5/middleware/value.go new file mode 100644 index 00000000..a9dfd434 --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/value.go @@ -0,0 +1,17 @@ +package middleware + +import ( + "context" + "net/http" +) + +// WithValue is a middleware that sets a given key/value in a context chain. +func WithValue(key, val interface{}) func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + r = r.WithContext(context.WithValue(r.Context(), key, val)) + next.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) + } +} diff --git a/vendor/github.com/go-chi/chi/v5/middleware/wrap_writer.go b/vendor/github.com/go-chi/chi/v5/middleware/wrap_writer.go new file mode 100644 index 00000000..cf5c44de --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/middleware/wrap_writer.go @@ -0,0 +1,219 @@ +package middleware + +// The original work was derived from Goji's middleware, source: +// https://github.com/zenazn/goji/tree/master/web/middleware + +import ( + "bufio" + "io" + "net" + "net/http" +) + +// NewWrapResponseWriter wraps an http.ResponseWriter, returning a proxy that allows you to +// hook into various parts of the response process. +func NewWrapResponseWriter(w http.ResponseWriter, protoMajor int) WrapResponseWriter { + _, fl := w.(http.Flusher) + + bw := basicWriter{ResponseWriter: w} + + if protoMajor == 2 { + _, ps := w.(http.Pusher) + if fl && ps { + return &http2FancyWriter{bw} + } + } else { + _, hj := w.(http.Hijacker) + _, rf := w.(io.ReaderFrom) + if fl && hj && rf { + return &httpFancyWriter{bw} + } + if fl && hj { + return &flushHijackWriter{bw} + } + if hj { + return &hijackWriter{bw} + } + } + + if fl { + return &flushWriter{bw} + } + + return &bw +} + +// WrapResponseWriter is a proxy around an http.ResponseWriter that allows you to hook +// into various parts of the response process. +type WrapResponseWriter interface { + http.ResponseWriter + // Status returns the HTTP status of the request, or 0 if one has not + // yet been sent. + Status() int + // BytesWritten returns the total number of bytes sent to the client. + BytesWritten() int + // Tee causes the response body to be written to the given io.Writer in + // addition to proxying the writes through. Only one io.Writer can be + // tee'd to at once: setting a second one will overwrite the first. + // Writes will be sent to the proxy before being written to this + // io.Writer. It is illegal for the tee'd writer to be modified + // concurrently with writes. + Tee(io.Writer) + // Unwrap returns the original proxied target. + Unwrap() http.ResponseWriter +} + +// basicWriter wraps a http.ResponseWriter that implements the minimal +// http.ResponseWriter interface. +type basicWriter struct { + http.ResponseWriter + wroteHeader bool + code int + bytes int + tee io.Writer +} + +func (b *basicWriter) WriteHeader(code int) { + if !b.wroteHeader { + b.code = code + b.wroteHeader = true + b.ResponseWriter.WriteHeader(code) + } +} + +func (b *basicWriter) Write(buf []byte) (int, error) { + b.maybeWriteHeader() + n, err := b.ResponseWriter.Write(buf) + if b.tee != nil { + _, err2 := b.tee.Write(buf[:n]) + // Prefer errors generated by the proxied writer. + if err == nil { + err = err2 + } + } + b.bytes += n + return n, err +} + +func (b *basicWriter) maybeWriteHeader() { + if !b.wroteHeader { + b.WriteHeader(http.StatusOK) + } +} + +func (b *basicWriter) Status() int { + return b.code +} + +func (b *basicWriter) BytesWritten() int { + return b.bytes +} + +func (b *basicWriter) Tee(w io.Writer) { + b.tee = w +} + +func (b *basicWriter) Unwrap() http.ResponseWriter { + return b.ResponseWriter +} + +// flushWriter ... +type flushWriter struct { + basicWriter +} + +func (f *flushWriter) Flush() { + f.wroteHeader = true + fl := f.basicWriter.ResponseWriter.(http.Flusher) + fl.Flush() +} + +var _ http.Flusher = &flushWriter{} + +// hijackWriter ... +type hijackWriter struct { + basicWriter +} + +func (f *hijackWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + hj := f.basicWriter.ResponseWriter.(http.Hijacker) + return hj.Hijack() +} + +var _ http.Hijacker = &hijackWriter{} + +// flushHijackWriter ... +type flushHijackWriter struct { + basicWriter +} + +func (f *flushHijackWriter) Flush() { + f.wroteHeader = true + fl := f.basicWriter.ResponseWriter.(http.Flusher) + fl.Flush() +} + +func (f *flushHijackWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + hj := f.basicWriter.ResponseWriter.(http.Hijacker) + return hj.Hijack() +} + +var _ http.Flusher = &flushHijackWriter{} +var _ http.Hijacker = &flushHijackWriter{} + +// httpFancyWriter is a HTTP writer that additionally satisfies +// http.Flusher, http.Hijacker, and io.ReaderFrom. It exists for the common case +// of wrapping the http.ResponseWriter that package http gives you, in order to +// make the proxied object support the full method set of the proxied object. +type httpFancyWriter struct { + basicWriter +} + +func (f *httpFancyWriter) Flush() { + f.wroteHeader = true + fl := f.basicWriter.ResponseWriter.(http.Flusher) + fl.Flush() +} + +func (f *httpFancyWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + hj := f.basicWriter.ResponseWriter.(http.Hijacker) + return hj.Hijack() +} + +func (f *http2FancyWriter) Push(target string, opts *http.PushOptions) error { + return f.basicWriter.ResponseWriter.(http.Pusher).Push(target, opts) +} + +func (f *httpFancyWriter) ReadFrom(r io.Reader) (int64, error) { + if f.basicWriter.tee != nil { + n, err := io.Copy(&f.basicWriter, r) + f.basicWriter.bytes += int(n) + return n, err + } + rf := f.basicWriter.ResponseWriter.(io.ReaderFrom) + f.basicWriter.maybeWriteHeader() + n, err := rf.ReadFrom(r) + f.basicWriter.bytes += int(n) + return n, err +} + +var _ http.Flusher = &httpFancyWriter{} +var _ http.Hijacker = &httpFancyWriter{} +var _ http.Pusher = &http2FancyWriter{} +var _ io.ReaderFrom = &httpFancyWriter{} + +// http2FancyWriter is a HTTP2 writer that additionally satisfies +// http.Flusher, and io.ReaderFrom. It exists for the common case +// of wrapping the http.ResponseWriter that package http gives you, in order to +// make the proxied object support the full method set of the proxied object. +type http2FancyWriter struct { + basicWriter +} + +func (f *http2FancyWriter) Flush() { + f.wroteHeader = true + fl := f.basicWriter.ResponseWriter.(http.Flusher) + fl.Flush() +} + +var _ http.Flusher = &http2FancyWriter{} diff --git a/vendor/github.com/go-chi/chi/v5/mux.go b/vendor/github.com/go-chi/chi/v5/mux.go new file mode 100644 index 00000000..977aa52d --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/mux.go @@ -0,0 +1,493 @@ +package chi + +import ( + "context" + "fmt" + "net/http" + "strings" + "sync" +) + +var _ Router = &Mux{} + +// Mux is a simple HTTP route multiplexer that parses a request path, +// records any URL params, and executes an end handler. It implements +// the http.Handler interface and is friendly with the standard library. +// +// Mux is designed to be fast, minimal and offer a powerful API for building +// modular and composable HTTP services with a large set of handlers. It's +// particularly useful for writing large REST API services that break a handler +// into many smaller parts composed of middlewares and end handlers. +type Mux struct { + // The computed mux handler made of the chained middleware stack and + // the tree router + handler http.Handler + + // The radix trie router + tree *node + + // Custom method not allowed handler + methodNotAllowedHandler http.HandlerFunc + + // A reference to the parent mux used by subrouters when mounting + // to a parent mux + parent *Mux + + // Routing context pool + pool *sync.Pool + + // Custom route not found handler + notFoundHandler http.HandlerFunc + + // The middleware stack + middlewares []func(http.Handler) http.Handler + + // Controls the behaviour of middleware chain generation when a mux + // is registered as an inline group inside another mux. + inline bool +} + +// NewMux returns a newly initialized Mux object that implements the Router +// interface. +func NewMux() *Mux { + mux := &Mux{tree: &node{}, pool: &sync.Pool{}} + mux.pool.New = func() interface{} { + return NewRouteContext() + } + return mux +} + +// ServeHTTP is the single method of the http.Handler interface that makes +// Mux interoperable with the standard library. It uses a sync.Pool to get and +// reuse routing contexts for each request. +func (mx *Mux) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Ensure the mux has some routes defined on the mux + if mx.handler == nil { + mx.NotFoundHandler().ServeHTTP(w, r) + return + } + + // Check if a routing context already exists from a parent router. + rctx, _ := r.Context().Value(RouteCtxKey).(*Context) + if rctx != nil { + mx.handler.ServeHTTP(w, r) + return + } + + // Fetch a RouteContext object from the sync pool, and call the computed + // mx.handler that is comprised of mx.middlewares + mx.routeHTTP. + // Once the request is finished, reset the routing context and put it back + // into the pool for reuse from another request. + rctx = mx.pool.Get().(*Context) + rctx.Reset() + rctx.Routes = mx + rctx.parentCtx = r.Context() + + // NOTE: r.WithContext() causes 2 allocations and context.WithValue() causes 1 allocation + r = r.WithContext(context.WithValue(r.Context(), RouteCtxKey, rctx)) + + // Serve the request and once its done, put the request context back in the sync pool + mx.handler.ServeHTTP(w, r) + mx.pool.Put(rctx) +} + +// Use appends a middleware handler to the Mux middleware stack. +// +// The middleware stack for any Mux will execute before searching for a matching +// route to a specific handler, which provides opportunity to respond early, +// change the course of the request execution, or set request-scoped values for +// the next http.Handler. +func (mx *Mux) Use(middlewares ...func(http.Handler) http.Handler) { + if mx.handler != nil { + panic("chi: all middlewares must be defined before routes on a mux") + } + mx.middlewares = append(mx.middlewares, middlewares...) +} + +// Handle adds the route `pattern` that matches any http method to +// execute the `handler` http.Handler. +func (mx *Mux) Handle(pattern string, handler http.Handler) { + mx.handle(mALL, pattern, handler) +} + +// HandleFunc adds the route `pattern` that matches any http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) HandleFunc(pattern string, handlerFn http.HandlerFunc) { + mx.handle(mALL, pattern, handlerFn) +} + +// Method adds the route `pattern` that matches `method` http method to +// execute the `handler` http.Handler. +func (mx *Mux) Method(method, pattern string, handler http.Handler) { + m, ok := methodMap[strings.ToUpper(method)] + if !ok { + panic(fmt.Sprintf("chi: '%s' http method is not supported.", method)) + } + mx.handle(m, pattern, handler) +} + +// MethodFunc adds the route `pattern` that matches `method` http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) MethodFunc(method, pattern string, handlerFn http.HandlerFunc) { + mx.Method(method, pattern, handlerFn) +} + +// Connect adds the route `pattern` that matches a CONNECT http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) Connect(pattern string, handlerFn http.HandlerFunc) { + mx.handle(mCONNECT, pattern, handlerFn) +} + +// Delete adds the route `pattern` that matches a DELETE http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) Delete(pattern string, handlerFn http.HandlerFunc) { + mx.handle(mDELETE, pattern, handlerFn) +} + +// Get adds the route `pattern` that matches a GET http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) Get(pattern string, handlerFn http.HandlerFunc) { + mx.handle(mGET, pattern, handlerFn) +} + +// Head adds the route `pattern` that matches a HEAD http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) Head(pattern string, handlerFn http.HandlerFunc) { + mx.handle(mHEAD, pattern, handlerFn) +} + +// Options adds the route `pattern` that matches an OPTIONS http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) Options(pattern string, handlerFn http.HandlerFunc) { + mx.handle(mOPTIONS, pattern, handlerFn) +} + +// Patch adds the route `pattern` that matches a PATCH http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) Patch(pattern string, handlerFn http.HandlerFunc) { + mx.handle(mPATCH, pattern, handlerFn) +} + +// Post adds the route `pattern` that matches a POST http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) Post(pattern string, handlerFn http.HandlerFunc) { + mx.handle(mPOST, pattern, handlerFn) +} + +// Put adds the route `pattern` that matches a PUT http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) Put(pattern string, handlerFn http.HandlerFunc) { + mx.handle(mPUT, pattern, handlerFn) +} + +// Trace adds the route `pattern` that matches a TRACE http method to +// execute the `handlerFn` http.HandlerFunc. +func (mx *Mux) Trace(pattern string, handlerFn http.HandlerFunc) { + mx.handle(mTRACE, pattern, handlerFn) +} + +// NotFound sets a custom http.HandlerFunc for routing paths that could +// not be found. The default 404 handler is `http.NotFound`. +func (mx *Mux) NotFound(handlerFn http.HandlerFunc) { + // Build NotFound handler chain + m := mx + hFn := handlerFn + if mx.inline && mx.parent != nil { + m = mx.parent + hFn = Chain(mx.middlewares...).HandlerFunc(hFn).ServeHTTP + } + + // Update the notFoundHandler from this point forward + m.notFoundHandler = hFn + m.updateSubRoutes(func(subMux *Mux) { + if subMux.notFoundHandler == nil { + subMux.NotFound(hFn) + } + }) +} + +// MethodNotAllowed sets a custom http.HandlerFunc for routing paths where the +// method is unresolved. The default handler returns a 405 with an empty body. +func (mx *Mux) MethodNotAllowed(handlerFn http.HandlerFunc) { + // Build MethodNotAllowed handler chain + m := mx + hFn := handlerFn + if mx.inline && mx.parent != nil { + m = mx.parent + hFn = Chain(mx.middlewares...).HandlerFunc(hFn).ServeHTTP + } + + // Update the methodNotAllowedHandler from this point forward + m.methodNotAllowedHandler = hFn + m.updateSubRoutes(func(subMux *Mux) { + if subMux.methodNotAllowedHandler == nil { + subMux.MethodNotAllowed(hFn) + } + }) +} + +// With adds inline middlewares for an endpoint handler. +func (mx *Mux) With(middlewares ...func(http.Handler) http.Handler) Router { + // Similarly as in handle(), we must build the mux handler once additional + // middleware registration isn't allowed for this stack, like now. + if !mx.inline && mx.handler == nil { + mx.updateRouteHandler() + } + + // Copy middlewares from parent inline muxs + var mws Middlewares + if mx.inline { + mws = make(Middlewares, len(mx.middlewares)) + copy(mws, mx.middlewares) + } + mws = append(mws, middlewares...) + + im := &Mux{ + pool: mx.pool, inline: true, parent: mx, tree: mx.tree, middlewares: mws, + notFoundHandler: mx.notFoundHandler, methodNotAllowedHandler: mx.methodNotAllowedHandler, + } + + return im +} + +// Group creates a new inline-Mux with a fresh middleware stack. It's useful +// for a group of handlers along the same routing path that use an additional +// set of middlewares. See _examples/. +func (mx *Mux) Group(fn func(r Router)) Router { + im := mx.With().(*Mux) + if fn != nil { + fn(im) + } + return im +} + +// Route creates a new Mux with a fresh middleware stack and mounts it +// along the `pattern` as a subrouter. Effectively, this is a short-hand +// call to Mount. See _examples/. +func (mx *Mux) Route(pattern string, fn func(r Router)) Router { + if fn == nil { + panic(fmt.Sprintf("chi: attempting to Route() a nil subrouter on '%s'", pattern)) + } + subRouter := NewRouter() + fn(subRouter) + mx.Mount(pattern, subRouter) + return subRouter +} + +// Mount attaches another http.Handler or chi Router as a subrouter along a routing +// path. It's very useful to split up a large API as many independent routers and +// compose them as a single service using Mount. See _examples/. +// +// Note that Mount() simply sets a wildcard along the `pattern` that will continue +// routing at the `handler`, which in most cases is another chi.Router. As a result, +// if you define two Mount() routes on the exact same pattern the mount will panic. +func (mx *Mux) Mount(pattern string, handler http.Handler) { + if handler == nil { + panic(fmt.Sprintf("chi: attempting to Mount() a nil handler on '%s'", pattern)) + } + + // Provide runtime safety for ensuring a pattern isn't mounted on an existing + // routing pattern. + if mx.tree.findPattern(pattern+"*") || mx.tree.findPattern(pattern+"/*") { + panic(fmt.Sprintf("chi: attempting to Mount() a handler on an existing path, '%s'", pattern)) + } + + // Assign sub-Router's with the parent not found & method not allowed handler if not specified. + subr, ok := handler.(*Mux) + if ok && subr.notFoundHandler == nil && mx.notFoundHandler != nil { + subr.NotFound(mx.notFoundHandler) + } + if ok && subr.methodNotAllowedHandler == nil && mx.methodNotAllowedHandler != nil { + subr.MethodNotAllowed(mx.methodNotAllowedHandler) + } + + mountHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rctx := RouteContext(r.Context()) + + // shift the url path past the previous subrouter + rctx.RoutePath = mx.nextRoutePath(rctx) + + // reset the wildcard URLParam which connects the subrouter + n := len(rctx.URLParams.Keys) - 1 + if n >= 0 && rctx.URLParams.Keys[n] == "*" && len(rctx.URLParams.Values) > n { + rctx.URLParams.Values[n] = "" + } + + handler.ServeHTTP(w, r) + }) + + if pattern == "" || pattern[len(pattern)-1] != '/' { + mx.handle(mALL|mSTUB, pattern, mountHandler) + mx.handle(mALL|mSTUB, pattern+"/", mountHandler) + pattern += "/" + } + + method := mALL + subroutes, _ := handler.(Routes) + if subroutes != nil { + method |= mSTUB + } + n := mx.handle(method, pattern+"*", mountHandler) + + if subroutes != nil { + n.subroutes = subroutes + } +} + +// Routes returns a slice of routing information from the tree, +// useful for traversing available routes of a router. +func (mx *Mux) Routes() []Route { + return mx.tree.routes() +} + +// Middlewares returns a slice of middleware handler functions. +func (mx *Mux) Middlewares() Middlewares { + return mx.middlewares +} + +// Match searches the routing tree for a handler that matches the method/path. +// It's similar to routing a http request, but without executing the handler +// thereafter. +// +// Note: the *Context state is updated during execution, so manage +// the state carefully or make a NewRouteContext(). +func (mx *Mux) Match(rctx *Context, method, path string) bool { + m, ok := methodMap[method] + if !ok { + return false + } + + node, _, h := mx.tree.FindRoute(rctx, m, path) + + if node != nil && node.subroutes != nil { + rctx.RoutePath = mx.nextRoutePath(rctx) + return node.subroutes.Match(rctx, method, rctx.RoutePath) + } + + return h != nil +} + +// NotFoundHandler returns the default Mux 404 responder whenever a route +// cannot be found. +func (mx *Mux) NotFoundHandler() http.HandlerFunc { + if mx.notFoundHandler != nil { + return mx.notFoundHandler + } + return http.NotFound +} + +// MethodNotAllowedHandler returns the default Mux 405 responder whenever +// a method cannot be resolved for a route. +func (mx *Mux) MethodNotAllowedHandler(methodsAllowed ...methodTyp) http.HandlerFunc { + if mx.methodNotAllowedHandler != nil { + return mx.methodNotAllowedHandler + } + return methodNotAllowedHandler(methodsAllowed...) +} + +// handle registers a http.Handler in the routing tree for a particular http method +// and routing pattern. +func (mx *Mux) handle(method methodTyp, pattern string, handler http.Handler) *node { + if len(pattern) == 0 || pattern[0] != '/' { + panic(fmt.Sprintf("chi: routing pattern must begin with '/' in '%s'", pattern)) + } + + // Build the computed routing handler for this routing pattern. + if !mx.inline && mx.handler == nil { + mx.updateRouteHandler() + } + + // Build endpoint handler with inline middlewares for the route + var h http.Handler + if mx.inline { + mx.handler = http.HandlerFunc(mx.routeHTTP) + h = Chain(mx.middlewares...).Handler(handler) + } else { + h = handler + } + + // Add the endpoint to the tree and return the node + return mx.tree.InsertRoute(method, pattern, h) +} + +// routeHTTP routes a http.Request through the Mux routing tree to serve +// the matching handler for a particular http method. +func (mx *Mux) routeHTTP(w http.ResponseWriter, r *http.Request) { + // Grab the route context object + rctx := r.Context().Value(RouteCtxKey).(*Context) + + // The request routing path + routePath := rctx.RoutePath + if routePath == "" { + if r.URL.RawPath != "" { + routePath = r.URL.RawPath + } else { + routePath = r.URL.Path + } + if routePath == "" { + routePath = "/" + } + } + + // Check if method is supported by chi + if rctx.RouteMethod == "" { + rctx.RouteMethod = r.Method + } + method, ok := methodMap[rctx.RouteMethod] + if !ok { + mx.MethodNotAllowedHandler().ServeHTTP(w, r) + return + } + + // Find the route + if _, _, h := mx.tree.FindRoute(rctx, method, routePath); h != nil { + h.ServeHTTP(w, r) + return + } + if rctx.methodNotAllowed { + mx.MethodNotAllowedHandler(rctx.methodsAllowed...).ServeHTTP(w, r) + } else { + mx.NotFoundHandler().ServeHTTP(w, r) + } +} + +func (mx *Mux) nextRoutePath(rctx *Context) string { + routePath := "/" + nx := len(rctx.routeParams.Keys) - 1 // index of last param in list + if nx >= 0 && rctx.routeParams.Keys[nx] == "*" && len(rctx.routeParams.Values) > nx { + routePath = "/" + rctx.routeParams.Values[nx] + } + return routePath +} + +// Recursively update data on child routers. +func (mx *Mux) updateSubRoutes(fn func(subMux *Mux)) { + for _, r := range mx.tree.routes() { + subMux, ok := r.SubRoutes.(*Mux) + if !ok { + continue + } + fn(subMux) + } +} + +// updateRouteHandler builds the single mux handler that is a chain of the middleware +// stack, as defined by calls to Use(), and the tree router (Mux) itself. After this +// point, no other middlewares can be registered on this Mux's stack. But you can still +// compose additional middlewares via Group()'s or using a chained middleware handler. +func (mx *Mux) updateRouteHandler() { + mx.handler = chain(mx.middlewares, http.HandlerFunc(mx.routeHTTP)) +} + +// methodNotAllowedHandler is a helper function to respond with a 405, +// method not allowed. It sets the Allow header with the list of allowed +// methods for the route. +func methodNotAllowedHandler(methodsAllowed ...methodTyp) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + for _, m := range methodsAllowed { + w.Header().Add("Allow", reverseMethodMap[m]) + } + w.WriteHeader(405) + w.Write(nil) + } +} diff --git a/vendor/github.com/go-chi/chi/v5/tree.go b/vendor/github.com/go-chi/chi/v5/tree.go new file mode 100644 index 00000000..c7d3bc57 --- /dev/null +++ b/vendor/github.com/go-chi/chi/v5/tree.go @@ -0,0 +1,892 @@ +package chi + +// Radix tree implementation below is a based on the original work by +// Armon Dadgar in https://github.com/armon/go-radix/blob/master/radix.go +// (MIT licensed). It's been heavily modified for use as a HTTP routing tree. + +import ( + "fmt" + "net/http" + "regexp" + "sort" + "strconv" + "strings" +) + +type methodTyp uint + +const ( + mSTUB methodTyp = 1 << iota + mCONNECT + mDELETE + mGET + mHEAD + mOPTIONS + mPATCH + mPOST + mPUT + mTRACE +) + +var mALL = mCONNECT | mDELETE | mGET | mHEAD | + mOPTIONS | mPATCH | mPOST | mPUT | mTRACE + +var methodMap = map[string]methodTyp{ + http.MethodConnect: mCONNECT, + http.MethodDelete: mDELETE, + http.MethodGet: mGET, + http.MethodHead: mHEAD, + http.MethodOptions: mOPTIONS, + http.MethodPatch: mPATCH, + http.MethodPost: mPOST, + http.MethodPut: mPUT, + http.MethodTrace: mTRACE, +} + +var reverseMethodMap = map[methodTyp]string{ + mCONNECT: http.MethodConnect, + mDELETE: http.MethodDelete, + mGET: http.MethodGet, + mHEAD: http.MethodHead, + mOPTIONS: http.MethodOptions, + mPATCH: http.MethodPatch, + mPOST: http.MethodPost, + mPUT: http.MethodPut, + mTRACE: http.MethodTrace, +} + +// RegisterMethod adds support for custom HTTP method handlers, available +// via Router#Method and Router#MethodFunc +func RegisterMethod(method string) { + if method == "" { + return + } + method = strings.ToUpper(method) + if _, ok := methodMap[method]; ok { + return + } + n := len(methodMap) + if n > strconv.IntSize-2 { + panic(fmt.Sprintf("chi: max number of methods reached (%d)", strconv.IntSize)) + } + mt := methodTyp(2 << n) + methodMap[method] = mt + mALL |= mt +} + +type nodeTyp uint8 + +const ( + ntStatic nodeTyp = iota // /home + ntRegexp // /{id:[0-9]+} + ntParam // /{user} + ntCatchAll // /api/v1/* +) + +type node struct { + // subroutes on the leaf node + subroutes Routes + + // regexp matcher for regexp nodes + rex *regexp.Regexp + + // HTTP handler endpoints on the leaf node + endpoints endpoints + + // prefix is the common prefix we ignore + prefix string + + // child nodes should be stored in-order for iteration, + // in groups of the node type. + children [ntCatchAll + 1]nodes + + // first byte of the child prefix + tail byte + + // node type: static, regexp, param, catchAll + typ nodeTyp + + // first byte of the prefix + label byte +} + +// endpoints is a mapping of http method constants to handlers +// for a given route. +type endpoints map[methodTyp]*endpoint + +type endpoint struct { + // endpoint handler + handler http.Handler + + // pattern is the routing pattern for handler nodes + pattern string + + // parameter keys recorded on handler nodes + paramKeys []string +} + +func (s endpoints) Value(method methodTyp) *endpoint { + mh, ok := s[method] + if !ok { + mh = &endpoint{} + s[method] = mh + } + return mh +} + +func (n *node) InsertRoute(method methodTyp, pattern string, handler http.Handler) *node { + var parent *node + search := pattern + + for { + // Handle key exhaustion + if len(search) == 0 { + // Insert or update the node's leaf handler + n.setEndpoint(method, handler, pattern) + return n + } + + // We're going to be searching for a wild node next, + // in this case, we need to get the tail + var label = search[0] + var segTail byte + var segEndIdx int + var segTyp nodeTyp + var segRexpat string + if label == '{' || label == '*' { + segTyp, _, segRexpat, segTail, _, segEndIdx = patNextSegment(search) + } + + var prefix string + if segTyp == ntRegexp { + prefix = segRexpat + } + + // Look for the edge to attach to + parent = n + n = n.getEdge(segTyp, label, segTail, prefix) + + // No edge, create one + if n == nil { + child := &node{label: label, tail: segTail, prefix: search} + hn := parent.addChild(child, search) + hn.setEndpoint(method, handler, pattern) + + return hn + } + + // Found an edge to match the pattern + + if n.typ > ntStatic { + // We found a param node, trim the param from the search path and continue. + // This param/wild pattern segment would already be on the tree from a previous + // call to addChild when creating a new node. + search = search[segEndIdx:] + continue + } + + // Static nodes fall below here. + // Determine longest prefix of the search key on match. + commonPrefix := longestPrefix(search, n.prefix) + if commonPrefix == len(n.prefix) { + // the common prefix is as long as the current node's prefix we're attempting to insert. + // keep the search going. + search = search[commonPrefix:] + continue + } + + // Split the node + child := &node{ + typ: ntStatic, + prefix: search[:commonPrefix], + } + parent.replaceChild(search[0], segTail, child) + + // Restore the existing node + n.label = n.prefix[commonPrefix] + n.prefix = n.prefix[commonPrefix:] + child.addChild(n, n.prefix) + + // If the new key is a subset, set the method/handler on this node and finish. + search = search[commonPrefix:] + if len(search) == 0 { + child.setEndpoint(method, handler, pattern) + return child + } + + // Create a new edge for the node + subchild := &node{ + typ: ntStatic, + label: search[0], + prefix: search, + } + hn := child.addChild(subchild, search) + hn.setEndpoint(method, handler, pattern) + return hn + } +} + +// addChild appends the new `child` node to the tree using the `pattern` as the trie key. +// For a URL router like chi's, we split the static, param, regexp and wildcard segments +// into different nodes. In addition, addChild will recursively call itself until every +// pattern segment is added to the url pattern tree as individual nodes, depending on type. +func (n *node) addChild(child *node, prefix string) *node { + search := prefix + + // handler leaf node added to the tree is the child. + // this may be overridden later down the flow + hn := child + + // Parse next segment + segTyp, _, segRexpat, segTail, segStartIdx, segEndIdx := patNextSegment(search) + + // Add child depending on next up segment + switch segTyp { + + case ntStatic: + // Search prefix is all static (that is, has no params in path) + // noop + + default: + // Search prefix contains a param, regexp or wildcard + + if segTyp == ntRegexp { + rex, err := regexp.Compile(segRexpat) + if err != nil { + panic(fmt.Sprintf("chi: invalid regexp pattern '%s' in route param", segRexpat)) + } + child.prefix = segRexpat + child.rex = rex + } + + if segStartIdx == 0 { + // Route starts with a param + child.typ = segTyp + + if segTyp == ntCatchAll { + segStartIdx = -1 + } else { + segStartIdx = segEndIdx + } + if segStartIdx < 0 { + segStartIdx = len(search) + } + child.tail = segTail // for params, we set the tail + + if segStartIdx != len(search) { + // add static edge for the remaining part, split the end. + // its not possible to have adjacent param nodes, so its certainly + // going to be a static node next. + + search = search[segStartIdx:] // advance search position + + nn := &node{ + typ: ntStatic, + label: search[0], + prefix: search, + } + hn = child.addChild(nn, search) + } + + } else if segStartIdx > 0 { + // Route has some param + + // starts with a static segment + child.typ = ntStatic + child.prefix = search[:segStartIdx] + child.rex = nil + + // add the param edge node + search = search[segStartIdx:] + + nn := &node{ + typ: segTyp, + label: search[0], + tail: segTail, + } + hn = child.addChild(nn, search) + + } + } + + n.children[child.typ] = append(n.children[child.typ], child) + n.children[child.typ].Sort() + return hn +} + +func (n *node) replaceChild(label, tail byte, child *node) { + for i := 0; i < len(n.children[child.typ]); i++ { + if n.children[child.typ][i].label == label && n.children[child.typ][i].tail == tail { + n.children[child.typ][i] = child + n.children[child.typ][i].label = label + n.children[child.typ][i].tail = tail + return + } + } + panic("chi: replacing missing child") +} + +func (n *node) getEdge(ntyp nodeTyp, label, tail byte, prefix string) *node { + nds := n.children[ntyp] + for i := 0; i < len(nds); i++ { + if nds[i].label == label && nds[i].tail == tail { + if ntyp == ntRegexp && nds[i].prefix != prefix { + continue + } + return nds[i] + } + } + return nil +} + +func (n *node) setEndpoint(method methodTyp, handler http.Handler, pattern string) { + // Set the handler for the method type on the node + if n.endpoints == nil { + n.endpoints = make(endpoints) + } + + paramKeys := patParamKeys(pattern) + + if method&mSTUB == mSTUB { + n.endpoints.Value(mSTUB).handler = handler + } + if method&mALL == mALL { + h := n.endpoints.Value(mALL) + h.handler = handler + h.pattern = pattern + h.paramKeys = paramKeys + for _, m := range methodMap { + h := n.endpoints.Value(m) + h.handler = handler + h.pattern = pattern + h.paramKeys = paramKeys + } + } else { + h := n.endpoints.Value(method) + h.handler = handler + h.pattern = pattern + h.paramKeys = paramKeys + } +} + +func (n *node) FindRoute(rctx *Context, method methodTyp, path string) (*node, endpoints, http.Handler) { + // Reset the context routing pattern and params + rctx.routePattern = "" + rctx.routeParams.Keys = rctx.routeParams.Keys[:0] + rctx.routeParams.Values = rctx.routeParams.Values[:0] + + // Find the routing handlers for the path + rn := n.findRoute(rctx, method, path) + if rn == nil { + return nil, nil, nil + } + + // Record the routing params in the request lifecycle + rctx.URLParams.Keys = append(rctx.URLParams.Keys, rctx.routeParams.Keys...) + rctx.URLParams.Values = append(rctx.URLParams.Values, rctx.routeParams.Values...) + + // Record the routing pattern in the request lifecycle + if rn.endpoints[method].pattern != "" { + rctx.routePattern = rn.endpoints[method].pattern + rctx.RoutePatterns = append(rctx.RoutePatterns, rctx.routePattern) + } + + return rn, rn.endpoints, rn.endpoints[method].handler +} + +// Recursive edge traversal by checking all nodeTyp groups along the way. +// It's like searching through a multi-dimensional radix trie. +func (n *node) findRoute(rctx *Context, method methodTyp, path string) *node { + nn := n + search := path + + for t, nds := range nn.children { + ntyp := nodeTyp(t) + if len(nds) == 0 { + continue + } + + var xn *node + xsearch := search + + var label byte + if search != "" { + label = search[0] + } + + switch ntyp { + case ntStatic: + xn = nds.findEdge(label) + if xn == nil || !strings.HasPrefix(xsearch, xn.prefix) { + continue + } + xsearch = xsearch[len(xn.prefix):] + + case ntParam, ntRegexp: + // short-circuit and return no matching route for empty param values + if xsearch == "" { + continue + } + + // serially loop through each node grouped by the tail delimiter + for idx := 0; idx < len(nds); idx++ { + xn = nds[idx] + + // label for param nodes is the delimiter byte + p := strings.IndexByte(xsearch, xn.tail) + + if p < 0 { + if xn.tail == '/' { + p = len(xsearch) + } else { + continue + } + } else if ntyp == ntRegexp && p == 0 { + continue + } + + if ntyp == ntRegexp && xn.rex != nil { + if !xn.rex.MatchString(xsearch[:p]) { + continue + } + } else if strings.IndexByte(xsearch[:p], '/') != -1 { + // avoid a match across path segments + continue + } + + prevlen := len(rctx.routeParams.Values) + rctx.routeParams.Values = append(rctx.routeParams.Values, xsearch[:p]) + xsearch = xsearch[p:] + + if len(xsearch) == 0 { + if xn.isLeaf() { + h := xn.endpoints[method] + if h != nil && h.handler != nil { + rctx.routeParams.Keys = append(rctx.routeParams.Keys, h.paramKeys...) + return xn + } + + for endpoints := range xn.endpoints { + if endpoints == mALL || endpoints == mSTUB { + continue + } + rctx.methodsAllowed = append(rctx.methodsAllowed, endpoints) + } + + // flag that the routing context found a route, but not a corresponding + // supported method + rctx.methodNotAllowed = true + } + } + + // recursively find the next node on this branch + fin := xn.findRoute(rctx, method, xsearch) + if fin != nil { + return fin + } + + // not found on this branch, reset vars + rctx.routeParams.Values = rctx.routeParams.Values[:prevlen] + xsearch = search + } + + rctx.routeParams.Values = append(rctx.routeParams.Values, "") + + default: + // catch-all nodes + rctx.routeParams.Values = append(rctx.routeParams.Values, search) + xn = nds[0] + xsearch = "" + } + + if xn == nil { + continue + } + + // did we find it yet? + if len(xsearch) == 0 { + if xn.isLeaf() { + h := xn.endpoints[method] + if h != nil && h.handler != nil { + rctx.routeParams.Keys = append(rctx.routeParams.Keys, h.paramKeys...) + return xn + } + + for endpoints := range xn.endpoints { + if endpoints == mALL || endpoints == mSTUB { + continue + } + rctx.methodsAllowed = append(rctx.methodsAllowed, endpoints) + } + + // flag that the routing context found a route, but not a corresponding + // supported method + rctx.methodNotAllowed = true + } + } + + // recursively find the next node.. + fin := xn.findRoute(rctx, method, xsearch) + if fin != nil { + return fin + } + + // Did not find final handler, let's remove the param here if it was set + if xn.typ > ntStatic { + if len(rctx.routeParams.Values) > 0 { + rctx.routeParams.Values = rctx.routeParams.Values[:len(rctx.routeParams.Values)-1] + } + } + + } + + return nil +} + +func (n *node) findEdge(ntyp nodeTyp, label byte) *node { + nds := n.children[ntyp] + num := len(nds) + idx := 0 + + switch ntyp { + case ntStatic, ntParam, ntRegexp: + i, j := 0, num-1 + for i <= j { + idx = i + (j-i)/2 + if label > nds[idx].label { + i = idx + 1 + } else if label < nds[idx].label { + j = idx - 1 + } else { + i = num // breaks cond + } + } + if nds[idx].label != label { + return nil + } + return nds[idx] + + default: // catch all + return nds[idx] + } +} + +func (n *node) isLeaf() bool { + return n.endpoints != nil +} + +func (n *node) findPattern(pattern string) bool { + nn := n + for _, nds := range nn.children { + if len(nds) == 0 { + continue + } + + n = nn.findEdge(nds[0].typ, pattern[0]) + if n == nil { + continue + } + + var idx int + var xpattern string + + switch n.typ { + case ntStatic: + idx = longestPrefix(pattern, n.prefix) + if idx < len(n.prefix) { + continue + } + + case ntParam, ntRegexp: + idx = strings.IndexByte(pattern, '}') + 1 + + case ntCatchAll: + idx = longestPrefix(pattern, "*") + + default: + panic("chi: unknown node type") + } + + xpattern = pattern[idx:] + if len(xpattern) == 0 { + return true + } + + return n.findPattern(xpattern) + } + return false +} + +func (n *node) routes() []Route { + rts := []Route{} + + n.walk(func(eps endpoints, subroutes Routes) bool { + if eps[mSTUB] != nil && eps[mSTUB].handler != nil && subroutes == nil { + return false + } + + // Group methodHandlers by unique patterns + pats := make(map[string]endpoints) + + for mt, h := range eps { + if h.pattern == "" { + continue + } + p, ok := pats[h.pattern] + if !ok { + p = endpoints{} + pats[h.pattern] = p + } + p[mt] = h + } + + for p, mh := range pats { + hs := make(map[string]http.Handler) + if mh[mALL] != nil && mh[mALL].handler != nil { + hs["*"] = mh[mALL].handler + } + + for mt, h := range mh { + if h.handler == nil { + continue + } + m := methodTypString(mt) + if m == "" { + continue + } + hs[m] = h.handler + } + + rt := Route{subroutes, hs, p} + rts = append(rts, rt) + } + + return false + }) + + return rts +} + +func (n *node) walk(fn func(eps endpoints, subroutes Routes) bool) bool { + // Visit the leaf values if any + if (n.endpoints != nil || n.subroutes != nil) && fn(n.endpoints, n.subroutes) { + return true + } + + // Recurse on the children + for _, ns := range n.children { + for _, cn := range ns { + if cn.walk(fn) { + return true + } + } + } + return false +} + +// patNextSegment returns the next segment details from a pattern: +// node type, param key, regexp string, param tail byte, param starting index, param ending index +func patNextSegment(pattern string) (nodeTyp, string, string, byte, int, int) { + ps := strings.Index(pattern, "{") + ws := strings.Index(pattern, "*") + + if ps < 0 && ws < 0 { + return ntStatic, "", "", 0, 0, len(pattern) // we return the entire thing + } + + // Sanity check + if ps >= 0 && ws >= 0 && ws < ps { + panic("chi: wildcard '*' must be the last pattern in a route, otherwise use a '{param}'") + } + + var tail byte = '/' // Default endpoint tail to / byte + + if ps >= 0 { + // Param/Regexp pattern is next + nt := ntParam + + // Read to closing } taking into account opens and closes in curl count (cc) + cc := 0 + pe := ps + for i, c := range pattern[ps:] { + if c == '{' { + cc++ + } else if c == '}' { + cc-- + if cc == 0 { + pe = ps + i + break + } + } + } + if pe == ps { + panic("chi: route param closing delimiter '}' is missing") + } + + key := pattern[ps+1 : pe] + pe++ // set end to next position + + if pe < len(pattern) { + tail = pattern[pe] + } + + var rexpat string + if idx := strings.Index(key, ":"); idx >= 0 { + nt = ntRegexp + rexpat = key[idx+1:] + key = key[:idx] + } + + if len(rexpat) > 0 { + if rexpat[0] != '^' { + rexpat = "^" + rexpat + } + if rexpat[len(rexpat)-1] != '$' { + rexpat += "$" + } + } + + return nt, key, rexpat, tail, ps, pe + } + + // Wildcard pattern as finale + if ws < len(pattern)-1 { + panic("chi: wildcard '*' must be the last value in a route. trim trailing text or use a '{param}' instead") + } + return ntCatchAll, "*", "", 0, ws, len(pattern) +} + +func patParamKeys(pattern string) []string { + pat := pattern + paramKeys := []string{} + for { + ptyp, paramKey, _, _, _, e := patNextSegment(pat) + if ptyp == ntStatic { + return paramKeys + } + for i := 0; i < len(paramKeys); i++ { + if paramKeys[i] == paramKey { + panic(fmt.Sprintf("chi: routing pattern '%s' contains duplicate param key, '%s'", pattern, paramKey)) + } + } + paramKeys = append(paramKeys, paramKey) + pat = pat[e:] + } +} + +// longestPrefix finds the length of the shared prefix +// of two strings +func longestPrefix(k1, k2 string) int { + max := len(k1) + if l := len(k2); l < max { + max = l + } + var i int + for i = 0; i < max; i++ { + if k1[i] != k2[i] { + break + } + } + return i +} + +func methodTypString(method methodTyp) string { + for s, t := range methodMap { + if method == t { + return s + } + } + return "" +} + +type nodes []*node + +// Sort the list of nodes by label +func (ns nodes) Sort() { sort.Sort(ns); ns.tailSort() } +func (ns nodes) Len() int { return len(ns) } +func (ns nodes) Swap(i, j int) { ns[i], ns[j] = ns[j], ns[i] } +func (ns nodes) Less(i, j int) bool { return ns[i].label < ns[j].label } + +// tailSort pushes nodes with '/' as the tail to the end of the list for param nodes. +// The list order determines the traversal order. +func (ns nodes) tailSort() { + for i := len(ns) - 1; i >= 0; i-- { + if ns[i].typ > ntStatic && ns[i].tail == '/' { + ns.Swap(i, len(ns)-1) + return + } + } +} + +func (ns nodes) findEdge(label byte) *node { + num := len(ns) + idx := 0 + i, j := 0, num-1 + for i <= j { + idx = i + (j-i)/2 + if label > ns[idx].label { + i = idx + 1 + } else if label < ns[idx].label { + j = idx - 1 + } else { + i = num // breaks cond + } + } + if ns[idx].label != label { + return nil + } + return ns[idx] +} + +// Route describes the details of a routing handler. +// Handlers map key is an HTTP method +type Route struct { + SubRoutes Routes + Handlers map[string]http.Handler + Pattern string +} + +// WalkFunc is the type of the function called for each method and route visited by Walk. +type WalkFunc func(method string, route string, handler http.Handler, middlewares ...func(http.Handler) http.Handler) error + +// Walk walks any router tree that implements Routes interface. +func Walk(r Routes, walkFn WalkFunc) error { + return walk(r, walkFn, "") +} + +func walk(r Routes, walkFn WalkFunc, parentRoute string, parentMw ...func(http.Handler) http.Handler) error { + for _, route := range r.Routes() { + mws := make([]func(http.Handler) http.Handler, len(parentMw)) + copy(mws, parentMw) + mws = append(mws, r.Middlewares()...) + + if route.SubRoutes != nil { + if err := walk(route.SubRoutes, walkFn, parentRoute+route.Pattern, mws...); err != nil { + return err + } + continue + } + + for method, handler := range route.Handlers { + if method == "*" { + // Ignore a "catchAll" method, since we pass down all the specific methods for each route. + continue + } + + fullRoute := parentRoute + route.Pattern + fullRoute = strings.Replace(fullRoute, "/*/", "/", -1) + + if chain, ok := handler.(*ChainHandler); ok { + if err := walkFn(method, fullRoute, chain.Endpoint, append(mws, chain.Middlewares...)...); err != nil { + return err + } + } else { + if err := walkFn(method, fullRoute, handler, mws...); err != nil { + return err + } + } + } + } + + return nil +} diff --git a/vendor/github.com/go-pkgz/email/.golangci.yml b/vendor/github.com/go-pkgz/email/.golangci.yml deleted file mode 100644 index ed5f8ddd..00000000 --- a/vendor/github.com/go-pkgz/email/.golangci.yml +++ /dev/null @@ -1,57 +0,0 @@ -run: - timeout: 5m - output: - format: tab - skip-dirs: - - vendor - -linters-settings: - govet: - check-shadowing: true - maligned: - suggest-new: true - goconst: - min-len: 2 - min-occurrences: 2 - misspell: - locale: US - lll: - line-length: 140 - gocritic: - enabled-tags: - - performance - - style - - experimental - disabled-checks: - - wrapperFunc - - hugeParam - - rangeValCopy - - singleCaseSwitch - - ifElseChain - -linters: - enable: - - dupl - - exportloopref - - gas - - gochecknoinits - - gocritic - - gocyclo - - gosimple - - govet - - ineffassign - - megacheck - - misspell - - nakedret - - prealloc - - revive - - stylecheck - - typecheck - - unconvert - - unparam - - unused - fast: false - disable-all: true - -issues: - exclude-use-default: false diff --git a/vendor/github.com/go-pkgz/email/LICENSE b/vendor/github.com/go-pkgz/email/LICENSE deleted file mode 100644 index 4203bbbb..00000000 --- a/vendor/github.com/go-pkgz/email/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2022 Umputun - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/vendor/github.com/go-pkgz/email/README.md b/vendor/github.com/go-pkgz/email/README.md deleted file mode 100644 index acee702c..00000000 --- a/vendor/github.com/go-pkgz/email/README.md +++ /dev/null @@ -1,71 +0,0 @@ -# email sending library - -[![Build Status](https://github.com/go-pkgz/email/workflows/build/badge.svg)](https://github.com/go-pkgz/email/actions) [![Coverage Status](https://coveralls.io/repos/github/go-pkgz/email/badge.svg?branch=master)](https://coveralls.io/github/go-pkgz/email?branch=master) [![Go Reference](https://pkg.go.dev/badge/github.com/go-pkgz/email.svg)](https://pkg.go.dev/github.com/go-pkgz/email) - -The library is a wrapper around the stdlib `net/smtp` simplifying email sending. It supports authentication, SSL/TLS, -user-specified SMTP servers, content-type, charset, multiple recipients and more. - -Usage example: - -```go -client := email.NewSender("localhost", email.ContentType("text/html"), email.Auth("user", "pass")) -err := client.Send("some content, foo bar", - email.Params{From: "me@example.com", To: []string{"to@example.com"}, Subject: "Hello world!", - Attachments: []string{"/path/to/file1.txt", "/path/to/file2.txt"}, - InlineImages: []string{"/path/to/image1.png", "/path/to/image2.png"}, - }) -``` - -## options - -`NewSender` accepts a number of options to configure the client: - -- `Port`: SMTP port (default: 25) -- `TLS`: Use TLS SMTP (default: false) -- `STARTTLS`: Use STARTTLS (default: false) -- `Auth(user, password)`: Username and password for SMTP authentication (default: empty, no authentication) -- `LoginAuth`: Use [LOGIN mechanism](https://www.ietf.org/archive/id/draft-murchison-sasl-login-00.txt) instead of PLAIN mechanism for SMTP authentication, e.g. this is relevant for Office 365 and Outlook.com -- `ContentType`: Content type for the email (default: "text/plain") -- `Charset`: Charset for the email (default: "utf-8") -- `TimeOut`: Timeout for the SMTP connection (default: 30 seconds) -- `Log`: Logger to use (default: no logging) -- `SMTP`: Set custom smtp client (default: none) - -See [go docs](https://pkg.go.dev/github.com/go-pkgz/email#Option) for `Option` functions. - -_Options should be passed to `NewSender` after the mandatory first (host) parameter._ - -## sending email - -To send email user need to create a sender first and then use `Send` method. The method accepts two parameters: - -- email content (string) -- parameters (`email.Params`) - ```go - type Params struct { - From string // From email field - To []string // From email field - Subject string // Email subject - UnsubscribeLink string // POST, https://support.google.com/mail/answer/81126 -> "Use one-click unsubscribe" - InReplyTo string // Identifier for email group (category), used for email grouping - Attachments []string // Attachments path - InlineImages []string // Embedding directly to email body. Autogenerated Content-Id (cid) equals to file name - } - ``` - -See [go docs](https://pkg.go.dev/github.com/go-pkgz/email#Sender.Send) for `Send` function. - -## technical details - -- Content-Transfer-Encoding set to `quoted-printable` -- Custom SMTP client (`smtp.Client` from stdlib) can be set by user with `SMTP` option. In this case it will be used instead of making a new smtp client internally. -- Logger can be set with `Log` option. It should implement `email.Logger` interface with a single `Logf(format string, args ...interface{})` method. By default, "no logging" internal logger is used. This interface is compatible with the `go-pkgz/lgr` logger. -- The library has no external dependencies, except for testing. It uses the stdlib `net/smtp` package. -- SSL/TLS supported with `TLS` option (usually on port 465) as well as with `STARTTLS` (usually on port 587). - -## limitations - -This library is not intended to be used for sending a lot of massive emails with -low latency requirements. The intended use case is sending simple messages, like alerts, notification and so on. -For example, sending alerts from a monitoring system, or for authentication-related emails, i.e. "password reset email", -"verification email", etc. diff --git a/vendor/github.com/go-pkgz/email/auth.go b/vendor/github.com/go-pkgz/email/auth.go deleted file mode 100644 index b1c8843a..00000000 --- a/vendor/github.com/go-pkgz/email/auth.go +++ /dev/null @@ -1,60 +0,0 @@ -package email - -import ( - "errors" - "net/smtp" -) - -// authMethod is SMTP authentication method -type authMethod string - -// List of supported authentication methods -const ( - authMethodPlain authMethod = "PLAIN" - authMethodLogin authMethod = "LOGIN" -) - -// newLoginAuth returns smtp.Auth that implements the LOGIN authentication -// mechanism as defined in the LOGIN SASL Mechanism document, -// https://www.ietf.org/archive/id/draft-murchison-sasl-login-00.txt. -// The returned smtp.Auth uses the given username and password to authenticate -// to the host. -// -// LOGIN will only send the credentials if the connection is using TLS -// or is connected to localhost. Otherwise authentication will fail with an -// error, without sending the credentials. -// -// LOGIN is described as obsolete in the SASL Mechanisms document -// but the mechanism is still in use, e.g. in Office 365 and Outlook.com. -func newLoginAuth(usr, pwd, host string) smtp.Auth { - return &loginAuth{usr, pwd, host} -} - -type loginAuth struct { - user string - password string - host string -} - -func isLocalhost(name string) bool { - return name == "localhost" || name == "127.0.0.1" || name == "::1" -} - -func (a *loginAuth) Start(server *smtp.ServerInfo) (proto string, toServer []byte, err error) { - if !server.TLS && !isLocalhost(server.Name) { - return "", nil, errors.New("unencrypted connection") - } - if server.Name != a.host { - return "", nil, errors.New("wrong host name") - } - - return "LOGIN", []byte(a.user), nil -} - -func (a *loginAuth) Next(fromServer []byte, more bool) (toServer []byte, err error) { - if more { - return []byte(a.password), nil - } - - return nil, nil -} diff --git a/vendor/github.com/go-pkgz/email/email.go b/vendor/github.com/go-pkgz/email/email.go deleted file mode 100644 index 033eb51d..00000000 --- a/vendor/github.com/go-pkgz/email/email.go +++ /dev/null @@ -1,366 +0,0 @@ -// Package email provides email sender -package email - -import ( - "bytes" - "crypto/tls" - "encoding/base64" - "errors" - "fmt" - "io" - "mime" - "mime/multipart" - "mime/quotedprintable" - "net" - "net/http" - "net/smtp" - "net/textproto" - "os" - "path/filepath" - "strings" - "time" -) - -//go:generate moq -out mocks/smpt_client.go -pkg mocks -skip-ensure -fmt goimports . SMTPClient -//go:generate moq -out mocks/logger.go -pkg mocks -skip-ensure -fmt goimports . Logger - -// Sender implements email sender -type Sender struct { - smtpClient SMTPClient - logger Logger - host string // SMTP host - port int // SMTP port - contentType string // Content type, optional. Will trigger MIME and Content-Type headers - tls bool // TLS auth - starttls bool // StartTLS - smtpUserName string // username - smtpPassword string // password - authMethod authMethod // auth method - timeOut time.Duration - contentCharset string - timeNow func() time.Time -} - -// Params contains all user-defined parameters to send emails -type Params struct { - From string // From email field - To []string // From email field - Subject string // Email subject - UnsubscribeLink string // POST, https://support.google.com/mail/answer/81126 -> "Use one-click unsubscribe" - InReplyTo string // Identifier for email group (category), used for email grouping - Attachments []string // Attachments path - InlineImages []string // InlineImages images path -} - -// Logger is used to log errors and debug messages -type Logger interface { - Logf(format string, args ...interface{}) -} - -// SMTPClient interface defines subset of net/smtp used by email client -type SMTPClient interface { - Mail(from string) error - Auth(auth smtp.Auth) error - Rcpt(to string) error - Data() (io.WriteCloser, error) - Quit() error - Close() error -} - -// NewSender creates email client with prepared smtp -func NewSender(smtpHost string, options ...Option) *Sender { - res := Sender{ - smtpClient: nil, - logger: nopLogger{}, - host: smtpHost, - port: 25, - contentType: `text/plain`, - tls: false, - smtpUserName: "", - smtpPassword: "", - authMethod: authMethodPlain, - contentCharset: "UTF-8", - timeOut: time.Second * 30, - timeNow: time.Now, - } - for _, opt := range options { - opt(&res) - } - - res.logger.Logf("[INFO] new email sender created with host: %s:%d, tls: %v, username: %q, timeout: %v, "+ - "content type: %q, charset: %q", smtpHost, - res.port, res.tls, res.smtpUserName, res.timeOut, res.contentType, res.contentCharset) - return &res -} - -// Send email with given text -// If SMTPClient defined in Email struct it will be used, if not - new smtp.Client on each send. -// Always closes client on completion or failure. -func (em *Sender) Send(text string, params Params) error { - em.logger.Logf("[DEBUG] send %q to %v", text, params.To) - - client := em.smtpClient - if client == nil { // if client not set make new net/smtp - c, err := em.client() - if err != nil { - return fmt.Errorf("failed to make smtp client: %w", err) - } - client = c - } - - var quit bool - defer func() { - if quit || client == nil { // quit set if Quit() call passed because it's closing connection as well. - return - } - if err := client.Close(); err != nil { - em.logger.Logf("[WARN] can't close smtp connection, %v", err) - } - }() - - if len(params.To) == 0 { - return errors.New("no recipients") - } - - if auth := em.auth(); auth != nil { - if err := client.Auth(auth); err != nil { - return fmt.Errorf("failed to auth to smtp %s:%d, %w", em.host, em.port, err) - } - } - - if err := client.Mail(params.From); err != nil { - return fmt.Errorf("bad from address %q: %w", params.From, err) - } - - for _, rcpt := range params.To { - if err := client.Rcpt(rcpt); err != nil { - return fmt.Errorf("bad to address %q: %w", params.To, err) - } - } - - writer, err := client.Data() - if err != nil { - return fmt.Errorf("can't make email writer: %w", err) - } - - msg, err := em.buildMessage(text, params) - if err != nil { - return fmt.Errorf("can't make email message: %w", err) - } - buf := bytes.NewBufferString(msg) - if _, err = buf.WriteTo(writer); err != nil { - return fmt.Errorf("failed to send email body to %q: %w", params.To, err) - } - if err = writer.Close(); err != nil { - em.logger.Logf("[WARN] can't close smtp body writer, %v", err) - } - - if err = client.Quit(); err != nil { - em.logger.Logf("[WARN] failed to send quit command to %s:%d, %v", em.host, em.port, err) - } else { - quit = true - } - return nil -} - -func (em *Sender) String() string { - return fmt.Sprintf("smtp://%s:%d, auth:%v, tls:%v, starttls:%v, timeout:%v, content-type:%q, charset:%q", - em.host, em.port, em.smtpUserName != "", em.tls, em.starttls, em.timeOut, em.contentType, em.contentCharset) -} - -func (em *Sender) client() (c *smtp.Client, err error) { - srvAddress := fmt.Sprintf("%s:%d", em.host, em.port) - tlsConf := &tls.Config{ - InsecureSkipVerify: false, - ServerName: em.host, - MinVersion: tls.VersionTLS12, - } - - if em.tls { - conn, e := tls.DialWithDialer(&net.Dialer{Timeout: em.timeOut}, "tcp", srvAddress, tlsConf) - if e != nil { - return nil, fmt.Errorf("failed to dial smtp tls to %s: %w", srvAddress, e) - } - if c, err = smtp.NewClient(conn, em.host); err != nil { - return nil, fmt.Errorf("failed to make smtp client for %s: %w", srvAddress, err) - } - return c, nil - } - - conn, err := net.DialTimeout("tcp", srvAddress, em.timeOut) - if err != nil { - return nil, fmt.Errorf("timeout connecting to %s: %w", srvAddress, err) - } - - c, err = smtp.NewClient(conn, em.host) - if err != nil { - return nil, fmt.Errorf("failed to dial: %w", err) - } - - if em.starttls { - if err = c.StartTLS(tlsConf); err != nil { - return nil, fmt.Errorf("failed to start tls: %w", err) - } - } - - return c, nil -} - -// auth returns an smtp.Auth that implements SMTP authentication mechanism -// depends on Sender settings. -func (em *Sender) auth() smtp.Auth { - if em.smtpUserName == "" || em.smtpPassword == "" { - return nil // no auth - } - - if em.authMethod == authMethodLogin { - return newLoginAuth(em.smtpUserName, em.smtpPassword, em.host) - } - return smtp.PlainAuth("", em.smtpUserName, em.smtpPassword, em.host) -} - -func (em *Sender) buildMessage(text string, params Params) (message string, err error) { - addHeader := func(msg, h, v string) string { - msg += fmt.Sprintf("%s: %s\n", h, v) - return msg - } - message = addHeader(message, "From", params.From) - message = addHeader(message, "To", strings.Join(params.To, ",")) - message = addHeader(message, "Subject", mime.BEncoding.Encode("utf-8", params.Subject)) - - if params.UnsubscribeLink != "" { - message = addHeader(message, "List-Unsubscribe-Post", "List-Unsubscribe=One-Click") - message = addHeader(message, "List-Unsubscribe", "<"+params.UnsubscribeLink+">") - } - - if params.InReplyTo != "" { - message = addHeader(message, "In-reply-to", "<"+params.InReplyTo+">") - } - - withAttachments := len(params.Attachments) > 0 - withInlineImg := len(params.InlineImages) > 0 - - if em.contentType != "" || withAttachments || withInlineImg { - message = addHeader(message, "MIME-version", "1.0") - } - - message = addHeader(message, "Date", em.timeNow().Format(time.RFC1123Z)) - - buff := &bytes.Buffer{} - qp := quotedprintable.NewWriter(buff) - mpMixed := multipart.NewWriter(buff) - boundaryMixed := mpMixed.Boundary() - mpRelated := multipart.NewWriter(buff) - boundaryRelated := mpRelated.Boundary() - - if withAttachments { - message = addHeader(message, "Content-Type", fmt.Sprintf("multipart/mixed; boundary=%q\r\n\r\n%s\r", - boundaryMixed, "--"+boundaryMixed)) - } - - if withInlineImg { - message = addHeader(message, "Content-Type", fmt.Sprintf("multipart/related; boundary=%q\r\n\r\n%s\r", - boundaryRelated, "--"+boundaryRelated)) - } - - if em.contentType != "" { - message = addHeader(message, "Content-Transfer-Encoding", "quoted-printable") - message = addHeader(message, "Content-Type", fmt.Sprintf("%s; charset=%q", em.contentType, em.contentCharset)) - - } - - if err := em.writeBody(qp, text); err != nil { - return "", fmt.Errorf("failed to write body: %w", err) - } - - if withInlineImg { - buff.WriteString("\r\n\r\n") - if err := em.writeFiles(mpRelated, params.InlineImages, "inline"); err != nil { - return "", fmt.Errorf("failed to write inline images: %w", err) - } - } - - if withAttachments { - buff.WriteString("\r\n\r\n") - if err := em.writeFiles(mpMixed, params.Attachments, "attachment"); err != nil { - return "", fmt.Errorf("failed to write attachments: %w", err) - } - } - - m := buff.String() - message += "\n" + m - // returns base part of the file location - return message, nil -} - -func (em *Sender) writeBody(wc io.WriteCloser, text string) error { - if _, err := wc.Write([]byte(text)); err != nil { - return err - } - if err := wc.Close(); err != nil { - return err - } - return nil -} - -func (em *Sender) writeFiles(mp *multipart.Writer, files []string, disposition string) error { - for _, attachment := range files { - file, err := os.Open(filepath.Clean(attachment)) - if err != nil { - return err - } - - // we need first 512 bytes to detect file type - fTypeBuff := make([]byte, 512) - _, err = file.Read(fTypeBuff) - if err != nil { - return fmt.Errorf("failed to read file type %q: %w", attachment, err) - } - - // remove null bytes in case file less than 512 bytes - fTypeBuff = bytes.Trim(fTypeBuff, "\x00") - fName := filepath.Base(attachment) - header := textproto.MIMEHeader{} - header.Set("Content-Type", http.DetectContentType(fTypeBuff)+"; name=\""+fName+"\"") - header.Set("Content-Transfer-Encoding", "base64") - - switch disposition { - case "attachment": - header.Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", fName)) - case "inline": - header.Set("Content-Disposition", fmt.Sprintf("inline; filename=%q", fName)) - header.Set("Content-ID", fmt.Sprintf("<%s>", fName)) - } - - writer, err := mp.CreatePart(header) - if err != nil { - return err - } - - // set reader offset at the beginning of the file because we read first 512 bytes - _, err = file.Seek(0, io.SeekStart) - if err != nil { - return err - } - - encoder := base64.NewEncoder(base64.StdEncoding, writer) - if _, err := io.Copy(encoder, file); err != nil { - return err - } - if err := encoder.Close(); err != nil { - return err - } - - if err := file.Close(); err != nil { - return err - } - } - if err := mp.Close(); err != nil { - return err - } - return nil -} - -type nopLogger struct{} - -func (nopLogger) Logf(format string, args ...interface{}) {} diff --git a/vendor/github.com/go-pkgz/email/options.go b/vendor/github.com/go-pkgz/email/options.go deleted file mode 100644 index 07070304..00000000 --- a/vendor/github.com/go-pkgz/email/options.go +++ /dev/null @@ -1,77 +0,0 @@ -package email - -import "time" - -// Option func type -type Option func(s *Sender) - -// SMTP sets SMTP client -func SMTP(smtp SMTPClient) Option { - return func(s *Sender) { - s.smtpClient = smtp - } -} - -// Log sets the logger for the email package -func Log(l Logger) Option { - return func(s *Sender) { - s.logger = l - } -} - -// Port sets SMTP port -func Port(port int) Option { - return func(s *Sender) { - s.port = port - } -} - -// ContentType sets content type of the email -func ContentType(contentType string) Option { - return func(s *Sender) { - s.contentType = contentType - } -} - -// Charset sets content charset of the email -func Charset(charset string) Option { - return func(s *Sender) { - s.contentCharset = charset - } -} - -// TLS enables TLS support -func TLS(enabled bool) Option { - return func(s *Sender) { - s.tls = enabled - } -} - -// STARTTLS enables STARTTLS support -func STARTTLS(enabled bool) Option { - return func(s *Sender) { - s.starttls = enabled - } -} - -// Auth sets smtp username and password -func Auth(smtpUserName, smtpPasswd string) Option { - return func(s *Sender) { - s.smtpUserName = smtpUserName - s.smtpPassword = smtpPasswd - } -} - -// LoginAuth sets LOGIN auth method -func LoginAuth() Option { - return func(s *Sender) { - s.authMethod = authMethodLogin - } -} - -// TimeOut sets smtp timeout -func TimeOut(timeOut time.Duration) Option { - return func(s *Sender) { - s.timeOut = timeOut - } -} diff --git a/vendor/github.com/go-pkgz/email/.gitignore b/vendor/github.com/go-pkgz/expirable-cache/.gitignore similarity index 100% rename from vendor/github.com/go-pkgz/email/.gitignore rename to vendor/github.com/go-pkgz/expirable-cache/.gitignore diff --git a/vendor/github.com/go-pkgz/repeater/.golangci.yml b/vendor/github.com/go-pkgz/expirable-cache/.golangci.yml similarity index 71% rename from vendor/github.com/go-pkgz/repeater/.golangci.yml rename to vendor/github.com/go-pkgz/expirable-cache/.golangci.yml index 989af86e..7442cd40 100644 --- a/vendor/github.com/go-pkgz/repeater/.golangci.yml +++ b/vendor/github.com/go-pkgz/expirable-cache/.golangci.yml @@ -7,8 +7,6 @@ linters-settings: min-complexity: 15 maligned: suggest-new: true - dupl: - threshold: 100 goconst: min-len: 2 min-occurrences: 2 @@ -25,9 +23,9 @@ linters-settings: - wrapperFunc linters: - disable-all: true enable: - megacheck + - golint - govet - unconvert - megacheck @@ -42,19 +40,25 @@ linters: - typecheck - ineffassign - varcheck + - stylecheck + - gochecknoinits + - scopelint + - gocritic + - nakedret + - gosimple + - prealloc fast: false - + disable-all: true run: -# modules-download-mode: vendor + output: + format: tab skip-dirs: - vendor issues: exclude-rules: - - text: "weak cryptographic primitive" + - text: "should have a package comment, unless it's in another file for this package" linters: - - gosec - -service: - golangci-lint-version: 1.16.x \ No newline at end of file + - golint + exclude-use-default: false diff --git a/vendor/github.com/go-pkgz/notify/LICENSE b/vendor/github.com/go-pkgz/expirable-cache/LICENSE similarity index 94% rename from vendor/github.com/go-pkgz/notify/LICENSE rename to vendor/github.com/go-pkgz/expirable-cache/LICENSE index 682ea0c0..d4771b09 100644 --- a/vendor/github.com/go-pkgz/notify/LICENSE +++ b/vendor/github.com/go-pkgz/expirable-cache/LICENSE @@ -1,6 +1,7 @@ MIT License -Copyright (c) 2021 Umputun +Copyright (c) 2020 Umputun +Copyright (c) 2020 Dmitry Verhoturov Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/vendor/github.com/go-pkgz/expirable-cache/README.md b/vendor/github.com/go-pkgz/expirable-cache/README.md new file mode 100644 index 00000000..e0ab7b7c --- /dev/null +++ b/vendor/github.com/go-pkgz/expirable-cache/README.md @@ -0,0 +1,70 @@ +# expirable-cache + +[![Build Status](https://github.com/go-pkgz/expirable-cache/workflows/build/badge.svg)](https://github.com/go-pkgz/expirable-cache/actions) +[![Coverage Status](https://coveralls.io/repos/github/go-pkgz/expirable-cache/badge.svg?branch=master)](https://coveralls.io/github/go-pkgz/expirable-cache?branch=master) +[![godoc](https://godoc.org/github.com/go-pkgz/expirable-cache?status.svg)](https://pkg.go.dev/github.com/go-pkgz/expirable-cache?tab=doc) + +Package cache implements expirable cache. + +- Support LRC, LRU and TTL-based eviction. +- Package is thread-safe and doesn't spawn any goroutines. +- On every Set() call, cache deletes single oldest entry in case it's expired. +- In case MaxSize is set, cache deletes the oldest entry disregarding its expiration date to maintain the size, +either using LRC or LRU eviction. +- In case of default TTL (10 years) and default MaxSize (0, unlimited) the cache will be truly unlimited + and will never delete entries from itself automatically. + +**Important**: only reliable way of not having expired entries stuck in a cache is to +run cache.DeleteExpired periodically using [time.Ticker](https://golang.org/pkg/time/#Ticker), +advisable period is 1/2 of TTL. + +This cache is heavily inspired by [hashicorp/golang-lru](https://github.com/hashicorp/golang-lru) _simplelru_ implementation. + +### Usage example + +```go +package main + +import ( + "fmt" + "time" + + "github.com/go-pkgz/expirable-cache" +) + +func main() { + // make cache with short TTL and 3 max keys + c, _ := cache.NewCache(cache.MaxKeys(3), cache.TTL(time.Millisecond*10)) + + // set value under key1. + // with 0 ttl (last parameter) will use cache-wide setting instead (10ms). + c.Set("key1", "val1", 0) + + // get value under key1 + r, ok := c.Get("key1") + + // check for OK value, because otherwise return would be nil and + // type conversion will panic + if ok { + rstr := r.(string) // convert cached value from interface{} to real type + fmt.Printf("value before expiration is found: %v, value: %v\n", ok, rstr) + } + + time.Sleep(time.Millisecond * 11) + + // get value under key1 after key expiration + r, ok = c.Get("key1") + // don't convert to string as with ok == false value would be nil + fmt.Printf("value after expiration is found: %v, value: %v\n", ok, r) + + // set value under key2, would evict old entry because it is already expired. + // ttl (last parameter) overrides cache-wide ttl. + c.Set("key2", "val2", time.Minute*5) + + fmt.Printf("%+v\n", c) + // Output: + // value before expiration is found: true, value: val1 + // value after expiration is found: false, value: + // Size: 1, Stats: {Hits:1 Misses:1 Added:2 Evicted:1} (50.0%) +} +``` \ No newline at end of file diff --git a/vendor/github.com/go-pkgz/expirable-cache/cache.go b/vendor/github.com/go-pkgz/expirable-cache/cache.go new file mode 100644 index 00000000..5966a8b0 --- /dev/null +++ b/vendor/github.com/go-pkgz/expirable-cache/cache.go @@ -0,0 +1,272 @@ +// Package cache implements Cache similar to hashicorp/golang-lru +// +// Support LRC, LRU and TTL-based eviction. +// Package is thread-safe and doesn't spawn any goroutines. +// On every Set() call, cache deletes single oldest entry in case it's expired. +// In case MaxSize is set, cache deletes the oldest entry disregarding its expiration date to maintain the size, +// either using LRC or LRU eviction. +// In case of default TTL (10 years) and default MaxSize (0, unlimited) the cache will be truly unlimited +// and will never delete entries from itself automatically. +// +// Important: only reliable way of not having expired entries stuck in a cache is to +// run cache.DeleteExpired periodically using time.Ticker, advisable period is 1/2 of TTL. +package cache + +import ( + "container/list" + "fmt" + "sync" + "time" +) + +// Cache defines cache interface +type Cache interface { + fmt.Stringer + Set(key string, value interface{}, ttl time.Duration) + Get(key string) (interface{}, bool) + Peek(key string) (interface{}, bool) + Keys() []string + Len() int + Invalidate(key string) + InvalidateFn(fn func(key string) bool) + RemoveOldest() + DeleteExpired() + Purge() + Stat() Stats +} + +// Stats provides statistics for cache +type Stats struct { + Hits, Misses int // cache effectiveness + Added, Evicted int // number of added and evicted records +} + +// cacheImpl provides Cache interface implementation. +type cacheImpl struct { + ttl time.Duration + maxKeys int + isLRU bool + onEvicted func(key string, value interface{}) + + sync.Mutex + stat Stats + items map[string]*list.Element + evictList *list.List +} + +// noEvictionTTL - very long ttl to prevent eviction +const noEvictionTTL = time.Hour * 24 * 365 * 10 + +// NewCache returns a new Cache. +// Default MaxKeys is unlimited (0). +// Default TTL is 10 years, sane value for expirable cache is 5 minutes. +// Default eviction mode is LRC, appropriate option allow to change it to LRU. +func NewCache(options ...Option) (Cache, error) { + res := cacheImpl{ + items: map[string]*list.Element{}, + evictList: list.New(), + ttl: noEvictionTTL, + maxKeys: 0, + } + + for _, opt := range options { + if err := opt(&res); err != nil { + return nil, fmt.Errorf("failed to set cache option: %w", err) + } + } + return &res, nil +} + +// Set key, ttl of 0 would use cache-wide TTL +func (c *cacheImpl) Set(key string, value interface{}, ttl time.Duration) { + c.Lock() + defer c.Unlock() + now := time.Now() + if ttl == 0 { + ttl = c.ttl + } + + // Check for existing item + if ent, ok := c.items[key]; ok { + c.evictList.MoveToFront(ent) + ent.Value.(*cacheItem).value = value + ent.Value.(*cacheItem).expiresAt = now.Add(ttl) + return + } + + // Add new item + ent := &cacheItem{key: key, value: value, expiresAt: now.Add(ttl)} + entry := c.evictList.PushFront(ent) + c.items[key] = entry + c.stat.Added++ + + // Remove oldest entry if it is expired, only in case of non-default TTL. + if c.ttl != noEvictionTTL || ttl != noEvictionTTL { + c.removeOldestIfExpired() + } + + // Verify size not exceeded + if c.maxKeys > 0 && len(c.items) > c.maxKeys { + c.removeOldest() + } +} + +// Get returns the key value if it's not expired +func (c *cacheImpl) Get(key string) (interface{}, bool) { + c.Lock() + defer c.Unlock() + if ent, ok := c.items[key]; ok { + // Expired item check + if time.Now().After(ent.Value.(*cacheItem).expiresAt) { + c.stat.Misses++ + return nil, false + } + if c.isLRU { + c.evictList.MoveToFront(ent) + } + c.stat.Hits++ + return ent.Value.(*cacheItem).value, true + } + c.stat.Misses++ + return nil, false +} + +// Peek returns the key value (or undefined if not found) without updating the "recently used"-ness of the key. +// Works exactly the same as Get in case of LRC mode (default one). +func (c *cacheImpl) Peek(key string) (interface{}, bool) { + c.Lock() + defer c.Unlock() + if ent, ok := c.items[key]; ok { + // Expired item check + if time.Now().After(ent.Value.(*cacheItem).expiresAt) { + c.stat.Misses++ + return nil, false + } + c.stat.Hits++ + return ent.Value.(*cacheItem).value, true + } + c.stat.Misses++ + return nil, false +} + +// Keys returns a slice of the keys in the cache, from oldest to newest. +func (c *cacheImpl) Keys() []string { + c.Lock() + defer c.Unlock() + return c.keys() +} + +// Len return count of items in cache, including expired +func (c *cacheImpl) Len() int { + c.Lock() + defer c.Unlock() + return c.evictList.Len() +} + +// Invalidate key (item) from the cache +func (c *cacheImpl) Invalidate(key string) { + c.Lock() + defer c.Unlock() + if ent, ok := c.items[key]; ok { + c.removeElement(ent) + } +} + +// InvalidateFn deletes multiple keys if predicate is true +func (c *cacheImpl) InvalidateFn(fn func(key string) bool) { + c.Lock() + defer c.Unlock() + for key, ent := range c.items { + if fn(key) { + c.removeElement(ent) + } + } +} + +// RemoveOldest remove oldest element in the cache +func (c *cacheImpl) RemoveOldest() { + c.Lock() + defer c.Unlock() + c.removeOldest() +} + +// DeleteExpired clears cache of expired items +func (c *cacheImpl) DeleteExpired() { + c.Lock() + defer c.Unlock() + for _, key := range c.keys() { + if time.Now().After(c.items[key].Value.(*cacheItem).expiresAt) { + c.removeElement(c.items[key]) + } + } +} + +// Purge clears the cache completely. +func (c *cacheImpl) Purge() { + c.Lock() + defer c.Unlock() + for k, v := range c.items { + delete(c.items, k) + c.stat.Evicted++ + if c.onEvicted != nil { + c.onEvicted(k, v.Value.(*cacheItem).value) + } + } + c.evictList.Init() +} + +// Stat gets the current stats for cache +func (c *cacheImpl) Stat() Stats { + c.Lock() + defer c.Unlock() + return c.stat +} + +func (c *cacheImpl) String() string { + stats := c.Stat() + size := c.Len() + return fmt.Sprintf("Size: %d, Stats: %+v (%0.1f%%)", size, stats, 100*float64(stats.Hits)/float64(stats.Hits+stats.Misses)) +} + +// Keys returns a slice of the keys in the cache, from oldest to newest. Has to be called with lock! +func (c *cacheImpl) keys() []string { + keys := make([]string, 0, len(c.items)) + for ent := c.evictList.Back(); ent != nil; ent = ent.Prev() { + keys = append(keys, ent.Value.(*cacheItem).key) + } + return keys +} + +// removeOldest removes the oldest item from the cache. Has to be called with lock! +func (c *cacheImpl) removeOldest() { + ent := c.evictList.Back() + if ent != nil { + c.removeElement(ent) + } +} + +// removeOldest removes the oldest item from the cache in case it's already expired. Has to be called with lock! +func (c *cacheImpl) removeOldestIfExpired() { + ent := c.evictList.Back() + if ent != nil && time.Now().After(ent.Value.(*cacheItem).expiresAt) { + c.removeElement(ent) + } +} + +// removeElement is used to remove a given list element from the cache. Has to be called with lock! +func (c *cacheImpl) removeElement(e *list.Element) { + c.evictList.Remove(e) + kv := e.Value.(*cacheItem) + delete(c.items, kv.key) + c.stat.Evicted++ + if c.onEvicted != nil { + c.onEvicted(kv.key, kv.value) + } +} + +// cacheItem is used to hold a value in the evictList +type cacheItem struct { + expiresAt time.Time + key string + value interface{} +} diff --git a/vendor/github.com/go-pkgz/expirable-cache/options.go b/vendor/github.com/go-pkgz/expirable-cache/options.go new file mode 100644 index 00000000..924345a0 --- /dev/null +++ b/vendor/github.com/go-pkgz/expirable-cache/options.go @@ -0,0 +1,40 @@ +package cache + +import "time" + +// Option func type +type Option func(lc *cacheImpl) error + +// OnEvicted called automatically for automatically and manually deleted entries +func OnEvicted(fn func(key string, value interface{})) Option { + return func(lc *cacheImpl) error { + lc.onEvicted = fn + return nil + } +} + +// MaxKeys functional option defines how many keys to keep. +// By default it is 0, which means unlimited. +func MaxKeys(max int) Option { + return func(lc *cacheImpl) error { + lc.maxKeys = max + return nil + } +} + +// TTL functional option defines TTL for all cache entries. +// By default it is set to 10 years, sane option for expirable cache might be 5 minutes. +func TTL(ttl time.Duration) Option { + return func(lc *cacheImpl) error { + lc.ttl = ttl + return nil + } +} + +// LRU sets cache to LRU (Least Recently Used) eviction mode. +func LRU() Option { + return func(lc *cacheImpl) error { + lc.isLRU = true + return nil + } +} diff --git a/vendor/github.com/go-pkgz/notify/README.md b/vendor/github.com/go-pkgz/notify/README.md deleted file mode 100644 index ae995fae..00000000 --- a/vendor/github.com/go-pkgz/notify/README.md +++ /dev/null @@ -1,202 +0,0 @@ -# Notify - -[![Build Status](https://github.com/go-pkgz/notify/workflows/build/badge.svg)](https://github.com/go-pkgz/notify/actions) [![Coverage Status](https://coveralls.io/repos/github/go-pkgz/notify/badge.svg?branch=master)](https://coveralls.io/github/go-pkgz/notify?branch=master) [![Go Reference](https://pkg.go.dev/badge/github.com/go-pkgz/notify.svg)](https://pkg.go.dev/github.com/go-pkgz/notify) - -This library provides ability to send notifications using multiple services: - -- Email -- Telegram -- Slack -- Webhook - -## Install - -`go get -u github.com/go-pkgz/notify` - -## Usage - -All supported notification methods could adhere to the following interface. Example on how to use it: - -```go -package main - -import ( - "context" - "fmt" - - "github.com/go-pkgz/notify" -) - -func main() { - // create notifiers - notifiers := []notify.Notifier{ - notify.NewWebhook(notify.WebhookParams{}), - notify.NewEmail(notify.SMTPParams{}), - notify.NewSlack("token"), - notify.NewTelegram(notify.TelegramParams{token: "token"}), - } - err := notify.Send(context.Background(), notifiers, "https://example.com/webhook", "Hello, world!") - if err != nil { - fmt.Printf("Sent message error: %s", err)) - } -} -``` - -### Email - -`mailto:` [scheme](https://datatracker.ietf.org/doc/html/rfc6068) is supported. Only `subject` and `from` query params are used. - -Examples: - -- `mailto:"John Wayne"?subject=test-subj&from="Notifier"` -- `mailto:addr1@example.org,addr2@example.org?&subject=test-subj&from=notify@example.org` - -```go -package main - -import ( - "context" - "log" - "time" - - "github.com/go-pkgz/notify" -) - -func main() { - wh := notify.NewEmail(notify.SMTPParams{ - Host: "localhost", // the only required field, others are optional - Port: 25, - TLS: false, // TLS, but not STARTTLS - ContentType: "text/html", - Charset: "UTF-8", - Username: "username", - Password: "password", - TimeOut: time.Second * 10, // default is 30 seconds - }) - err := wh.Send( - context.Background(), - `mailto:"John Wayne"?subject=test-subj&from="Notifier"`, - "Hello, World!", - ) - if err != nil { - log.Fatalf("problem sending message using email, %v", err) - } -} -``` - -### Telegram - -`telegram:` scheme akin to `mailto:` is supported. Query params `parseMode` ([doc](https://core.telegram.org/bots/api#formatting-options), legacy `Markdown` by default, preferable use `MarkdownV2` or `HTML` instead). Examples: - -- `telegram:channel` -- `telegram:channelID` // channel ID is a number, like `-1001480738202`: use [that instruction](https://remark42.com/docs/configuration/telegram/#notifications-for-administrators) to obtain it -- `telegram:userID` - -[Here](https://remark42.com/docs/configuration/telegram/#getting-bot-token-for-telegram) is an instruction on obtaining token for your notification bot. - -```go -package main - -import ( - "context" - "log" - - "github.com/go-pkgz/notify" -) - -func main() { - tg := notify.NewTelegram(notify.TelegramParams{ - Token: "token", // required - Timeout: time.Second * 10, // default is 5 seconds - SuccessMsg: // optional, for auth, set by default - ErrorMsg: // optional, for auth, unset by default - }) - err := tg.Send(context.Background(), "telegram:-1001480738202", "Hello, World!") - if err != nil { - log.Fatalf("problem sending message using telegram, %v", err) - } -} -``` - -#### HTML Formatting - -parseMode `HTML` supports [limited set of tags](https://core.telegram.org/bots/api#html-style), so `Telegram` provides `TelegramSupportedHTML` method which strips all unsupported tags and replaces `h1-h3` with `` and `h4-h6` with `` to preserve formatting. - -If you want to post text into HTML tag like text, you can use `EscapeTelegramText` method to escape it (by replacing symbols `&`, `<`, `>` with `&`, `<`, `>`). - -#### Authorisation - -You can use Telegram notifications as described above, just to send messages. But also, you can use `Telegram` to authorise users as a login method or to sign them up for notifications. Functions used for processing updates from users are `GetBotUsername`, `AddToken`, `CheckToken`, `Request`, and `Run` or `ProcessUpdate` (only one of two can be used at a time). - -Normal flow is following: -1. you run the `Run` goroutine -2. call `AddToken` and provide user with that token -3. user clicks on the link `https://t.me//?start=` -4. you call `CheckToken` to verify that the user clicked the link, and if so you will receive user's UserID - -Alternative flow is the same, but instead of running the `Run` goroutine, you set up process update flow separately (to use with [auth](https://github.com/go-pkgz/auth/blob/master/provider/telegram.go) as well, for example) and run `ProcessUpdate` when update is received. Example of such a setup can be seen [in Remark42](https://github.com/umputun/remark42/blob/c027dcd/backend/app/providers/telegram.go). - -### Slack - -`slack:` scheme akin to `mailto:` is supported. `title`, `titleLink`, `attachmentText` and query params are used: if they are defined, message would be sent with a [text attachment](https://api.slack.com/reference/messaging/attachments). Examples: - -- `slack:channel` -- `slack:channelID` -- `slack:userID` -- `slack:channel?title=title&attachmentText=test%20text&titleLink=https://example.org` - -```go -package main - -import ( - "context" - "log" - - "github.com/go-pkgz/notify" - "github.com/slack-go/slack" -) - -func main() { - wh := notify.NewSlack( - "token", - slack.OptionDebug(true), // optional, you can pass any slack.Options - ) - err := wh.Send(context.Background(), "slack:general", "Hello, World!") - if err != nil { - log.Fatalf("problem sending message using slack, %v", err) - } -} -``` - -### Webhook - -`http://` and `https://` schemas are supported. - -```go -package main - -import ( - "context" - "log" - "time" - - "github.com/go-pkgz/notify" -) - -func main() { - wh := notify.NewWebhook(notify.WebhookParams{ - Timeout: time.Second, // optional, default is 5 seconds - Headers: []string{"Content-Type:application/json,text/plain"}, // optional - }) - err := wh.Send(context.Background(), "https://example.org/webhook", "Hello, World!") - if err != nil { - log.Fatalf("problem sending message using webhook, %v", err) - } -} -``` - -## Status - -The library extracted from [remark42](https://github.com/umputun/remark) project. The original code in production use on multiple sites and seems to work fine. - -`go-pkgz/notify` library still in development and until version 1 released some breaking changes possible. diff --git a/vendor/github.com/go-pkgz/notify/email.go b/vendor/github.com/go-pkgz/notify/email.go deleted file mode 100644 index ff18c212..00000000 --- a/vendor/github.com/go-pkgz/notify/email.go +++ /dev/null @@ -1,146 +0,0 @@ -package notify - -import ( - "context" - "fmt" - "net/mail" - "net/url" - "strings" - "time" - - "github.com/go-pkgz/email" -) - -// SMTPParams contain settings for smtp server connection -type SMTPParams struct { - Host string // SMTP host - Port int // SMTP port - TLS bool // TLS auth - StartTLS bool // StartTLS auth - ContentType string // Content type - Charset string // Character set - LoginAuth bool // LOGIN auth method instead of default PLAIN, needed for Office 365 and outlook.com - Username string // username - Password string // password - TimeOut time.Duration // TCP connection timeout -} - -// Email notifications client -type Email struct { - SMTPParams - sender *email.Sender -} - -// NewEmail makes new Email object -func NewEmail(smtpParams SMTPParams) *Email { - var opts []email.Option - - if smtpParams.Username != "" { - opts = append(opts, email.Auth(smtpParams.Username, smtpParams.Password)) - } - - if smtpParams.ContentType != "" { - opts = append(opts, email.ContentType(smtpParams.ContentType)) - } - - if smtpParams.Charset != "" { - opts = append(opts, email.Charset(smtpParams.Charset)) - } - - if smtpParams.LoginAuth { - opts = append(opts, email.LoginAuth()) - } - - if smtpParams.Port != 0 { - opts = append(opts, email.Port(smtpParams.Port)) - } - - if smtpParams.TimeOut != 0 { - opts = append(opts, email.TimeOut(smtpParams.TimeOut)) - } - - if smtpParams.TLS { - opts = append(opts, email.TLS(true)) - } - - if smtpParams.StartTLS { - opts = append(opts, email.STARTTLS(true)) - } - - sender := email.NewSender(smtpParams.Host, opts...) - - return &Email{sender: sender, SMTPParams: smtpParams} -} - -// Send sends the message over Email, with "from", "subject" and "unsubscribeLink" parsed from destination field -// with "mailto:" schema. -// "unsubscribeLink" passed as a header, https://support.google.com/mail/answer/81126 -> "Use one-click unsubscribe" -// -// Example: -// -// - mailto:"John Wayne"?subject=test-subj&from="Notifier" -// - mailto:addr1@example.org,addr2@example.org?subject=test-subj&from=notify@example.org&unsubscribeLink=http://example.org/unsubscribe -func (e *Email) Send(ctx context.Context, destination, text string) error { - emailParams, err := e.parseDestination(destination) - if err != nil { - return fmt.Errorf("problem parsing destination: %w", err) - } - - select { - case <-ctx.Done(): - return ctx.Err() - default: - return e.sender.Send(text, emailParams) - } -} - -// Schema returns schema prefix supported by this client -func (e *Email) Schema() string { - return "mailto" -} - -// String representation of Email object -func (e *Email) String() string { - str := fmt.Sprintf("email: with username '%s' at server %s:%d", e.Username, e.Host, e.Port) - if e.TLS { - str += " with TLS" - } - if e.StartTLS { - str += " with StartTLS" - } - return str -} - -// parses "mailto:" URL and returns email parameters -func (e *Email) parseDestination(destination string) (email.Params, error) { - // parse URL - u, err := url.Parse(destination) - if err != nil { - return email.Params{}, err - } - if u.Scheme != "mailto" { - return email.Params{}, fmt.Errorf("unsupported scheme %s, should be mailto", u.Scheme) - } - - // parse destination address(es) - addresses, err := mail.ParseAddressList(u.Opaque) - if err != nil { - return email.Params{}, fmt.Errorf("problem parsing email recipients: %w", err) - } - destinations := []string{} - for _, addr := range addresses { - stringAddr := addr.String() - // in case of mailgun, correct RFC5322 address with <> yield 501 error, so we need to remove brackets - if strings.HasPrefix(stringAddr, "<") && strings.HasSuffix(stringAddr, ">") { - stringAddr = stringAddr[1 : len(stringAddr)-1] - } - destinations = append(destinations, stringAddr) - } - - return email.Params{ - From: u.Query().Get("from"), - To: destinations, - Subject: u.Query().Get("subject"), - UnsubscribeLink: u.Query().Get("unsubscribeLink"), - }, nil -} diff --git a/vendor/github.com/go-pkgz/notify/interface.go b/vendor/github.com/go-pkgz/notify/interface.go deleted file mode 100644 index b852e53d..00000000 --- a/vendor/github.com/go-pkgz/notify/interface.go +++ /dev/null @@ -1,28 +0,0 @@ -// Package notify provides notification functionality. -package notify - -import ( - "context" - "fmt" - "strings" -) - -// Notifier defines common interface among all notifiers -type Notifier interface { - fmt.Stringer - Schema() string // returns schema prefix supported by this client - Send(ctx context.Context, destination, text string) error // sends message to provided destination -} - -// Send sends message to provided destination, picking the right one based on destination schema -func Send(ctx context.Context, notifiers []Notifier, destination, text string) error { - for _, n := range notifiers { - if strings.HasPrefix(destination, n.Schema()) { - return n.Send(ctx, destination, text) - } - } - if strings.Contains(destination, ":") { - return fmt.Errorf("unsupported destination schema: %s", strings.Split(destination, ":")[0]) - } - return fmt.Errorf("unsupported destination schema: %s", destination) -} diff --git a/vendor/github.com/go-pkgz/notify/slack.go b/vendor/github.com/go-pkgz/notify/slack.go deleted file mode 100644 index 52ee3b45..00000000 --- a/vendor/github.com/go-pkgz/notify/slack.go +++ /dev/null @@ -1,108 +0,0 @@ -package notify - -import ( - "context" - "errors" - "fmt" - "net/url" - "strings" - - "github.com/slack-go/slack" -) - -// Slack notifications client -type Slack struct { - client *slack.Client -} - -// NewSlack makes Slack client for notifications -func NewSlack(token string, opts ...slack.Option) *Slack { - return &Slack{client: slack.New(token, opts...)} -} - -// Send sends the message over Slack, with "title", "titleLink" and "attachmentText" parsed from destination field -// with "slack:" schema same way "mailto:" schema is constructed. -// -// Example: -// -// - slack:channelName -// - slack:channelID -// - slack:userID -// - slack:channel?title=title&attachmentText=test%20text&titleLink=https://example.org -func (s *Slack) Send(ctx context.Context, destination, text string) error { - channelID, attachment, err := s.parseDestination(destination) - if err != nil { - return fmt.Errorf("problem parsing destination: %w", err) - } - options := []slack.MsgOption{slack.MsgOptionText(text, false)} - if attachment.Title != "" { - options = append(options, slack.MsgOptionAttachments(attachment)) - } - - select { - case <-ctx.Done(): - return ctx.Err() - default: - _, _, err = s.client.PostMessageContext(ctx, channelID, options...) - return err - } -} - -// Schema returns schema prefix supported by this client -func (s *Slack) Schema() string { - return "slack" -} - -func (s *Slack) String() string { - return "slack notifications destination" -} - -// parses "slack:" in a manner "mailto:" URL is parsed url and returns channelID and attachment. -// if channelID is channel name and not ID (starting with C for channel and with U for user), -// then it will be resolved to ID. -func (s *Slack) parseDestination(destination string) (string, slack.Attachment, error) { - // parse URL - u, err := url.Parse(destination) - if err != nil { - return "", slack.Attachment{}, err - } - if u.Scheme != "slack" { - return "", slack.Attachment{}, fmt.Errorf("unsupported scheme %s, should be slack", u.Scheme) - } - channelID := u.Opaque - if !strings.HasPrefix(u.Opaque, "C") && !strings.HasPrefix(u.Opaque, "U") { - channelID, err = s.findChannelIDByName(u.Opaque) - if err != nil { - return "", slack.Attachment{}, fmt.Errorf("problem retrieving channel ID for #%s: %w", u.Opaque, err) - } - } - - return channelID, - slack.Attachment{ - Title: u.Query().Get("title"), - TitleLink: u.Query().Get("titleLink"), - Text: u.Query().Get("attachmentText"), - }, nil -} - -func (s *Slack) findChannelIDByName(name string) (string, error) { - params := slack.GetConversationsParameters{} - for { - channels, next, err := s.client.GetConversations(¶ms) - if err != nil { - return "", err - } - - for _, channel := range channels { - if channel.Name == name { - return channel.ID, nil - } - } - - if next == "" { - break - } - params.Cursor = next - } - return "", errors.New("no such channel") -} diff --git a/vendor/github.com/go-pkgz/notify/telegram.go b/vendor/github.com/go-pkgz/notify/telegram.go deleted file mode 100644 index a8a412a9..00000000 --- a/vendor/github.com/go-pkgz/notify/telegram.go +++ /dev/null @@ -1,479 +0,0 @@ -package notify - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - neturl "net/url" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" - - log "github.com/go-pkgz/lgr" - "github.com/go-pkgz/repeater" - "github.com/microcosm-cc/bluemonday" - "golang.org/x/net/html" -) - -// TelegramParams contain settings for telegram notifications -type TelegramParams struct { - Token string // token for telegram bot API interactions - Timeout time.Duration // http client timeout - ErrorMsg, SuccessMsg string // messages for successful and unsuccessful subscription requests to bot - - apiPrefix string // changed only in tests -} - -// Telegram notifications client -type Telegram struct { - TelegramParams - - // Identifier of the first update to be requested. - // Should be equal to LastSeenUpdateID + 1 - // See https://core.telegram.org/bots/api#getupdates - updateOffset int - apiPollInterval time.Duration // interval to check updates from Telegram API and answer to users - expiredCleanupInterval time.Duration // interval to check and clean up expired notification requests - username string // bot username - run int32 // non-zero if Run goroutine has started - requests struct { - sync.RWMutex - data map[string]tgAuthRequest - } -} - -// telegramMsg is used to send message through Telegram bot API -type telegramMsg struct { - Text string `json:"text"` - ParseMode string `json:"parse_mode,omitempty"` -} - -type tgAuthRequest struct { - confirmed bool // whether login request has been confirmed and user info set - expires time.Time - telegramID string - user string - site string -} - -// TelegramBotInfo structure contains information about telegram bot, which is used from whole telegram API response -type TelegramBotInfo struct { - Username string `json:"username"` -} - -const telegramTimeOut = 5000 * time.Millisecond -const telegramAPIPrefix = "https://api.telegram.org/bot" -const tgPollInterval = time.Second * 5 -const tgCleanupInterval = time.Minute * 5 - -// NewTelegram makes telegram bot for notifications -func NewTelegram(params TelegramParams) (*Telegram, error) { - res := Telegram{TelegramParams: params} - - if res.apiPrefix == "" { - res.apiPrefix = telegramAPIPrefix - } - if res.Timeout == 0 { - res.Timeout = telegramTimeOut - } - - if res.SuccessMsg == "" { - res.SuccessMsg = "✅ You have successfully authenticated, check the web!" - } - - res.apiPollInterval = tgPollInterval - res.expiredCleanupInterval = tgCleanupInterval - log.Printf("[DEBUG] create new telegram notifier for api=%s, timeout=%s", res.apiPrefix, res.Timeout) - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - botInfo, err := res.botInfo(ctx) - if err != nil { - return nil, fmt.Errorf("can't retrieve bot info from Telegram API: %w", err) - } - res.username = botInfo.Username - - res.requests.data = make(map[string]tgAuthRequest) - - return &res, nil -} - -// Send sends provided message to Telegram chat, with `parseMode` parsed from destination field (Markdown by default) -// with "telegram:" schema same way "mailto:" schema is constructed. -// -// Example: -// -// - telegram:channel -// - telegram:chatID // chatID is a number, like `-1001480738202` -// - telegram:channel?parseMode=HTML -func (t *Telegram) Send(ctx context.Context, destination, text string) error { - chatID, parseMode, err := t.parseDestination(destination) - if err != nil { - return fmt.Errorf("problem parsing destination: %w", err) - } - - body := telegramMsg{Text: text, ParseMode: parseMode} - b, err := json.Marshal(body) - if err != nil { - return err - } - - url := fmt.Sprintf("sendMessage?chat_id=%s&disable_web_page_preview=true", chatID) - return t.Request(ctx, url, b, &struct{}{}) -} - -// TelegramSupportedHTML returns HTML with only tags allowed in Telegram HTML message payload, also trims ending newlines -// -// https://core.telegram.org/bots/api#html-style -func TelegramSupportedHTML(htmlText string) string { - adjustedHTMLText := adjustHTMLTags(htmlText) - p := bluemonday.NewPolicy() - p.AllowElements("b", "strong", "i", "em", "u", "ins", "s", "strike", "del", "a", "code", "pre") - p.AllowAttrs("href").OnElements("a") - p.AllowAttrs("class").OnElements("code") - return strings.TrimRight(p.Sanitize(adjustedHTMLText), "\n") -} - -// EscapeTelegramText returns text sanitized of symbols not allowed inside other HTML tags in Telegram HTML message payload -// -// https://core.telegram.org/bots/api#html-style -func EscapeTelegramText(text string) string { - // order is important - text = strings.ReplaceAll(text, "&", "&") - text = strings.ReplaceAll(text, "<", "<") - text = strings.ReplaceAll(text, ">", ">") - return text -} - -// telegram not allow h1-h6 tags -// replace these tags with a combination of and for visual distinction -func adjustHTMLTags(htmlText string) string { - buff := strings.Builder{} - tokenizer := html.NewTokenizer(strings.NewReader(htmlText)) - for { - if tokenizer.Next() == html.ErrorToken { - return buff.String() - } - token := tokenizer.Token() - switch token.Type { - case html.StartTagToken, html.EndTagToken: - switch token.Data { - case "h1", "h2", "h3": - if token.Type == html.StartTagToken { - buff.WriteString("") - } - if token.Type == html.EndTagToken { - buff.WriteString("") - } - case "h4", "h5", "h6": - if token.Type == html.StartTagToken { - buff.WriteString("") - } - if token.Type == html.EndTagToken { - buff.WriteString("") - } - default: - buff.WriteString(token.String()) - } - default: - buff.WriteString(token.String()) - } - } -} - -// TelegramUpdate contains update information, which is used from whole telegram API response -type TelegramUpdate struct { - Result []struct { - UpdateID int `json:"update_id"` - Message struct { - Chat struct { - ID int `json:"id"` - Name string `json:"first_name"` - Type string `json:"type"` - } `json:"chat"` - Text string `json:"text"` - } `json:"message"` - } `json:"result"` -} - -// GetBotUsername returns bot username -func (t *Telegram) GetBotUsername() string { - return t.username -} - -// AddToken adds token -func (t *Telegram) AddToken(token, user, site string, expires time.Time) { - t.requests.Lock() - t.requests.data[token] = tgAuthRequest{ - expires: expires, - user: user, - site: site, - } - t.requests.Unlock() -} - -// CheckToken verifies incoming token, returns the user address if it's confirmed and empty string otherwise -func (t *Telegram) CheckToken(token, user string) (telegram, site string, err error) { - t.requests.RLock() - authRequest, ok := t.requests.data[token] - t.requests.RUnlock() - - if !ok { - return "", "", errors.New("request is not found") - } - - if time.Now().After(authRequest.expires) { - t.requests.Lock() - delete(t.requests.data, token) - t.requests.Unlock() - return "", "", errors.New("request expired") - } - - if !authRequest.confirmed { - return "", "", errors.New("request is not verified yet") - } - - if authRequest.user != user { - return "", "", errors.New("user does not match original requester") - } - - // Delete request - t.requests.Lock() - delete(t.requests.data, token) - t.requests.Unlock() - - return authRequest.telegramID, authRequest.site, nil -} - -// Run starts processing login requests sent in Telegram, required for user notifications to work -// Blocks caller -func (t *Telegram) Run(ctx context.Context) { - atomic.AddInt32(&t.run, 1) - processUpdatedTicker := time.NewTicker(t.apiPollInterval) - cleanupTicker := time.NewTicker(t.expiredCleanupInterval) - - for { - select { - case <-ctx.Done(): - processUpdatedTicker.Stop() - cleanupTicker.Stop() - atomic.AddInt32(&t.run, -1) - return - case <-processUpdatedTicker.C: - updates, err := t.getUpdates(ctx) - if err != nil { - log.Printf("[WARN] Error while getting telegram updates: %v", err) - continue - } - t.processUpdates(ctx, updates) - case <-cleanupTicker.C: - now := time.Now() - t.requests.Lock() - for key, req := range t.requests.data { - if now.After(req.expires) { - delete(t.requests.data, key) - } - } - t.requests.Unlock() - } - } -} - -// ProcessUpdate is alternative to Run, it processes provided plain text update from Telegram -// so that caller could get updates and send it not only there but to multiple sources -func (t *Telegram) ProcessUpdate(ctx context.Context, textUpdate string) error { - if atomic.LoadInt32(&t.run) != 0 { - return errors.New("the Run goroutine should not be used with ProcessUpdate") - } - defer func() { - // as Run goroutine is not running, clean up old requests on each update - // even if we hit json decode error - now := time.Now() - t.requests.Lock() - for key, req := range t.requests.data { - if now.After(req.expires) { - delete(t.requests.data, key) - } - } - t.requests.Unlock() - }() - var updates TelegramUpdate - if err := json.Unmarshal([]byte(textUpdate), &updates); err != nil { - return fmt.Errorf("failed to decode provided telegram update: %w", err) - } - t.processUpdates(ctx, &updates) - return nil -} - -// Schema returns schema prefix supported by this client -func (t *Telegram) Schema() string { - return "telegram" -} - -func (t *Telegram) String() string { - return "telegram notifications destination" -} - -// parses "telegram:" in a manner "mailto:" URL is parsed url and returns chatID and parseMode. -// if chatID is channel name and not a numerical ID, `@` will be added to it -func (t *Telegram) parseDestination(destination string) (chatID, parseMode string, err error) { - // parse URL - u, err := neturl.Parse(destination) - if err != nil { - return "", "", err - } - if u.Scheme != "telegram" { - return "", "", fmt.Errorf("unsupported scheme %s, should be telegram", u.Scheme) - } - - chatID = u.Opaque - if _, err := strconv.ParseInt(chatID, 10, 64); err != nil { - chatID = "@" + chatID // if chatID not a number enforce @ prefix - } - - parseMode = "Markdown" - if u.Query().Get("parseMode") != "" { - parseMode = u.Query().Get("parseMode") - } - - return chatID, parseMode, nil -} - -// getUpdates fetches incoming updates -func (t *Telegram) getUpdates(ctx context.Context) (*TelegramUpdate, error) { - url := `getUpdates?allowed_updates=["message"]` - if t.updateOffset != 0 { - url += fmt.Sprintf("&offset=%d", t.updateOffset) - } - - var result TelegramUpdate - - err := t.Request(ctx, url, nil, &result) - if err != nil { - return nil, fmt.Errorf("failed to fetch updates: %w", err) - } - - for _, u := range result.Result { - if u.UpdateID >= t.updateOffset { - t.updateOffset = u.UpdateID + 1 - } - } - - return &result, nil -} - -// processUpdates processes a batch of updates from telegram servers -func (t *Telegram) processUpdates(ctx context.Context, updates *TelegramUpdate) { - for _, update := range updates.Result { - if update.Message.Chat.Type != "private" { - continue - } - - if !strings.HasPrefix(update.Message.Text, "/start ") { - continue - } - - token := strings.TrimPrefix(update.Message.Text, "/start ") - - t.requests.RLock() - authRequest, ok := t.requests.data[token] - if !ok { // No such token - t.requests.RUnlock() - if t.ErrorMsg != "" { - if err := t.sendText(ctx, update.Message.Chat.ID, t.ErrorMsg); err != nil { - log.Printf("[WARN] failed to notify telegram peer: %v", err) - } - } - continue - } - t.requests.RUnlock() - - authRequest.confirmed = true - authRequest.telegramID = strconv.Itoa(update.Message.Chat.ID) - - t.requests.Lock() - t.requests.data[token] = authRequest - t.requests.Unlock() - - if err := t.sendText(ctx, update.Message.Chat.ID, t.SuccessMsg); err != nil { - log.Printf("[ERROR] failed to notify telegram peer: %v", err) - } - } -} - -// sendText sends a plain text message to telegram peer -func (t *Telegram) sendText(ctx context.Context, recipientID int, msg string) error { - url := fmt.Sprintf("sendMessage?chat_id=%d&text=%s", recipientID, neturl.PathEscape(msg)) - return t.Request(ctx, url, nil, &struct{}{}) -} - -// botInfo returns info about configured bot -func (t *Telegram) botInfo(ctx context.Context) (*TelegramBotInfo, error) { - var resp = struct { - Result *TelegramBotInfo `json:"result"` - }{} - - err := t.Request(ctx, "getMe", nil, &resp) - if err != nil { - return nil, err - } - if resp.Result == nil { - return nil, errors.New("received empty result") - } - - return resp.Result, nil -} - -// Request makes a request to the Telegram API and return the result -func (t *Telegram) Request(ctx context.Context, method string, b []byte, data interface{}) error { - return repeater.NewDefault(3, time.Millisecond*250).Do(ctx, func() error { - url := fmt.Sprintf("%s%s/%s", t.apiPrefix, t.Token, method) - - var req *http.Request - var err error - if b == nil { - req, err = http.NewRequestWithContext(ctx, "GET", url, http.NoBody) - } else { - req, err = http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(b)) - req.Header.Set("Content-Type", "application/json; charset=utf-8") - } - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - - client := http.Client{Timeout: t.Timeout} - defer client.CloseIdleConnections() - resp, err := client.Do(req) - if err != nil { - return fmt.Errorf("failed to send request: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return t.parseError(resp.Body, resp.StatusCode) - } - - if err = json.NewDecoder(resp.Body).Decode(data); err != nil { - return fmt.Errorf("failed to decode json response: %w", err) - } - - return nil - }) -} - -func (t *Telegram) parseError(r io.Reader, statusCode int) error { - tgErr := struct { - Description string `json:"description"` - }{} - if err := json.NewDecoder(r).Decode(&tgErr); err != nil { - return fmt.Errorf("unexpected telegram API status code %d", statusCode) - } - return fmt.Errorf("unexpected telegram API status code %d, error: %q", statusCode, tgErr.Description) -} diff --git a/vendor/github.com/go-pkgz/notify/webhook.go b/vendor/github.com/go-pkgz/notify/webhook.go deleted file mode 100644 index 6cdffeed..00000000 --- a/vendor/github.com/go-pkgz/notify/webhook.go +++ /dev/null @@ -1,96 +0,0 @@ -package notify - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "net/http" - "strings" - "time" -) - -const webhookTimeOut = 5000 * time.Millisecond - -// WebhookParams contain settings for webhook notifications -type WebhookParams struct { - Timeout time.Duration - Headers []string // headers in format "header:value" -} - -// Webhook notifications client -type Webhook struct { - WebhookParams - webhookClient webhookClient -} - -// webhookClient defines an interface of client for webhook -type webhookClient interface { - Do(*http.Request) (*http.Response, error) -} - -// NewWebhook makes Webhook -func NewWebhook(params WebhookParams) *Webhook { - res := &Webhook{WebhookParams: params} - - if res.Timeout == 0 { - res.Timeout = webhookTimeOut - } - - res.webhookClient = &http.Client{Timeout: res.Timeout} - - return res -} - -// Send sends Webhook notification. Destination field is expected to have http:// or https:// schema. -// -// Example: -// -// - https://example.com/webhook -func (wh *Webhook) Send(ctx context.Context, destination, text string) error { - payload := bytes.NewBufferString(text) - httpReq, err := http.NewRequestWithContext(ctx, "POST", destination, payload) - if err != nil { - return fmt.Errorf("unable to create webhook request: %w", err) - } - - for _, h := range wh.Headers { - elems := strings.Split(h, ":") - if len(elems) != 2 { - continue - } - httpReq.Header.Set(strings.TrimSpace(elems[0]), strings.TrimSpace(elems[1])) - } - - resp, err := wh.webhookClient.Do(httpReq) - if err != nil { - return fmt.Errorf("webhook request failed: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - errMsg := fmt.Sprintf("webhook request failed with non-OK status code: %d", resp.StatusCode) - respBody, e := io.ReadAll(resp.Body) - if e != nil { - return errors.New(errMsg) - } - return fmt.Errorf("%s, body: %s", errMsg, respBody) - } - - return nil -} - -// Schema returns schema prefix supported by this client -func (wh *Webhook) Schema() string { - return "http" -} - -// String describes the webhook instance -func (wh *Webhook) String() string { - str := fmt.Sprintf("webhook notification with timeout %s", wh.Timeout) - if wh.Headers != nil { - str += fmt.Sprintf(" and headers %v", wh.Headers) - } - return str -} diff --git a/vendor/github.com/go-pkgz/repeater/.travis.yml b/vendor/github.com/go-pkgz/repeater/.travis.yml deleted file mode 100644 index 697ce97f..00000000 --- a/vendor/github.com/go-pkgz/repeater/.travis.yml +++ /dev/null @@ -1,20 +0,0 @@ -language: go - -go: - - "1.12.x" - -install: true - -before_install: - - export TZ=America/Chicago - - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.13.2 - - go get github.com/mattn/goveralls - - export PATH=$(pwd)/bin:$PATH - -script: - - GO111MODULE=on go get ./... - - GO111MODULE=on go mod vendor - - GO111MODULE=on go test -v -mod=vendor -covermode=count -coverprofile=profile.cov ./... || travis_terminate 1; - - GO111MODULE=on go test -v -covermode=count -coverprofile=profile.cov ./... || travis_terminate 1; - - golangci-lint run || travis_terminate 1; - - $GOPATH/bin/goveralls -coverprofile=profile.cov -service=travis-ci diff --git a/vendor/github.com/go-pkgz/repeater/README.md b/vendor/github.com/go-pkgz/repeater/README.md deleted file mode 100644 index 76d8c906..00000000 --- a/vendor/github.com/go-pkgz/repeater/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Repeater [![Build Status](https://travis-ci.org/go-pkgz/repeater.svg?branch=master)](https://travis-ci.org/go-pkgz/repeater) [![Go Report Card](https://goreportcard.com/badge/github.com/go-pkgz/repeater)](https://goreportcard.com/report/github.com/go-pkgz/repeater) [![Coverage Status](https://coveralls.io/repos/github/go-pkgz/repeater/badge.svg?branch=master)](https://coveralls.io/github/go-pkgz/repeater?branch=master) - -Repeater calls a function until it returns no error, up to some number of iterations and delays defined by strategy. It terminates immediately on err from the provided (optional) list of critical errors. - -## Install and update - -`go get -u github.com/go-pkgz/repeater` - -## How to use - -New Repeater created by `New(strtg strategy.Interface)` or shortcut for defaults - `NewDefault(repeats int, delay time.Duration) *Repeater`. - -To activate invoke `Do` method. `Do` repeats func until no error returned. Predefined (optional) errors terminates the loop immediately. - -`func (r Repeater) Do(ctx context.Context, fun func() error, errors ...error) (err error)` - -### Repeating strategy - -User can provide his own strategy implementing the interface: - -```go -type Interface interface { - Start(ctx context.Context) chan struct{} -} -``` - -Returned channels used as "ticks," i.e., for each repeat or initial operation one read from this channel needed. Closing this channel indicates "done with retries." It is pretty much the same idea as `time.Timer` or `time.Tick` implements. Note - the first (technically not-repeated-yet) call won't happen **until something sent to the channel**. For this reason, the typical strategy sends the first "tick" before the first wait/sleep. - -Three most common strategies provided by package and ready to use: -1. **Fixed delay**, up to max number of attempts - `NewFixedDelay(repeats int, delay time.Duration)`. -It is the default strategy used by `repeater.NewDefault` constructor -2. **BackOff** with jitter provides exponential backoff. It starts from 100ms interval and goes in steps with `last * math.Pow(factor, attempt)`. Optional jitter randomizes intervals a little bit. The strategy created by `NewBackoff(repeats int, factor float64, jitter bool)`. _Factor = 1 effectively makes this strategy fixed with 100ms delay._ -3. **Once** strategy does not do any repeats and mainly used for tests/mocks - `NewOnce()` - - diff --git a/vendor/github.com/go-pkgz/repeater/repeater.go b/vendor/github.com/go-pkgz/repeater/repeater.go deleted file mode 100644 index e6370ce5..00000000 --- a/vendor/github.com/go-pkgz/repeater/repeater.go +++ /dev/null @@ -1,64 +0,0 @@ -// Package repeater call fun till it returns no error, up to repeat some number of iterations and delays defined by strategy. -// Repeats number and delays defined by strategy.Interface. Terminates immediately on err from -// provided, optional list of critical errors -package repeater - -import ( - "context" - "time" - - "github.com/go-pkgz/repeater/strategy" -) - -// Repeater is the main object, should be made by New or NewDefault, embeds strategy -type Repeater struct { - strategy.Interface -} - -// New repeater with a given strategy. If strategy=nil initializes with FixedDelay 5sec, 10 times. -func New(strtg strategy.Interface) *Repeater { - if strtg == nil { - strtg = &strategy.FixedDelay{Repeats: 10, Delay: time.Second * 5} - } - result := Repeater{Interface: strtg} - return &result -} - -// NewDefault makes repeater with FixedDelay strategy -func NewDefault(repeats int, delay time.Duration) *Repeater { - return New(&strategy.FixedDelay{Repeats: repeats, Delay: delay}) -} - -// Do repeats fun till no error. Predefined (optional) errors terminate immediately -func (r Repeater) Do(ctx context.Context, fun func() error, errors ...error) (err error) { - - ctx, cancelFunc := context.WithCancel(ctx) - defer cancelFunc() // ensure strategy's channel termination - - inErrors := func(err error) bool { - for _, e := range errors { - if e == err { - return true - } - } - return false - } - - ch := r.Start(ctx) // channel of ticks-like events provided by strategy - for { - select { - case <-ctx.Done(): - return ctx.Err() - case _, ok := <-ch: - if !ok { // closed channel indicates completion or early termination, set by strategy - return err - } - if err = fun(); err == nil { - return nil - } - if err != nil && inErrors(err) { // terminate on critical error from provided list - return err - } - } - } -} diff --git a/vendor/github.com/go-pkgz/repeater/strategy/backoff.go b/vendor/github.com/go-pkgz/repeater/strategy/backoff.go deleted file mode 100644 index 12f53b69..00000000 --- a/vendor/github.com/go-pkgz/repeater/strategy/backoff.go +++ /dev/null @@ -1,59 +0,0 @@ -package strategy - -import ( - "context" - "math" - "math/rand" - "sync" - "time" -) - -// Backoff implements strategy.Interface for exponential-backoff -// it starts from 100ms (by default, if no Duration set) and goes in steps with last * math.Pow(factor, attempt) -// optional jitter randomize intervals a little bit. -type Backoff struct { - Duration time.Duration - Repeats int - Factor float64 - Jitter bool - - once sync.Once -} - -// Start returns channel, similar to time.Timer -// then publishing signals to channel ch for retries attempt. Closed ch indicates "done" event -// consumer (repeater) should stop it explicitly after completion -func (b *Backoff) Start(ctx context.Context) <-chan struct{} { - - b.once.Do(func() { - if b.Duration == 0 { - b.Duration = 100 * time.Millisecond - } - if b.Repeats == 0 { - b.Repeats = 1 - } - if b.Factor <= 0 { - b.Factor = 1 - } - }) - - ch := make(chan struct{}) - go func() { - defer close(ch) - rnd := rand.New(rand.NewSource(int64(time.Now().Nanosecond()))) - for i := 0; i < b.Repeats; i++ { - select { - case <-ctx.Done(): - return - case ch <- struct{}{}: - } - - delay := float64(b.Duration) * math.Pow(b.Factor, float64(i)) - if b.Jitter { - delay = rnd.Float64()*(float64(2*b.Duration)) + (delay - float64(b.Duration)) - } - sleep(ctx, time.Duration(delay)) - } - }() - return ch -} diff --git a/vendor/github.com/go-pkgz/repeater/strategy/fixed.go b/vendor/github.com/go-pkgz/repeater/strategy/fixed.go deleted file mode 100644 index ddd283e8..00000000 --- a/vendor/github.com/go-pkgz/repeater/strategy/fixed.go +++ /dev/null @@ -1,36 +0,0 @@ -package strategy - -import ( - "context" - "time" -) - -// FixedDelay implements strategy.Interface for fixed intervals up to max repeats -type FixedDelay struct { - Repeats int - Delay time.Duration -} - -// Start returns channel, similar to time.Timer -// then publishing signals to channel ch for retries attempt. -// can be terminated (canceled) via context. -func (s *FixedDelay) Start(ctx context.Context) <-chan struct{} { - if s.Repeats == 0 { - s.Repeats = 1 - } - ch := make(chan struct{}) - go func() { - defer func() { - close(ch) - }() - for i := 0; i < s.Repeats; i++ { - select { - case <-ctx.Done(): - return - case ch <- struct{}{}: - } - sleep(ctx, s.Delay) - } - }() - return ch -} diff --git a/vendor/github.com/go-pkgz/repeater/strategy/strategy.go b/vendor/github.com/go-pkgz/repeater/strategy/strategy.go deleted file mode 100644 index 75af12ba..00000000 --- a/vendor/github.com/go-pkgz/repeater/strategy/strategy.go +++ /dev/null @@ -1,35 +0,0 @@ -// Package strategy defines repeater's strategy and implements some. -// Strategy result is a channel acting like time.Timer ot time.Tick -package strategy - -import ( - "context" - "time" -) - -// Interface for repeater strategy. Returns channel with ticks -type Interface interface { - Start(ctx context.Context) <-chan struct{} -} - -// Once strategy eliminate repeats and makes a single try only -type Once struct{} - -// Start returns closed channel with a single element to prevent any repeats -func (s *Once) Start(ctx context.Context) <-chan struct{} { - ch := make(chan struct{}) - go func() { - ch <- struct{}{} - close(ch) - }() - return ch -} - -func sleep(ctx context.Context, duration time.Duration) { - select { - case <-time.After(duration): - return - case <-ctx.Done(): - return - } -} diff --git a/vendor/github.com/go-pkgz/repeater/.gitignore b/vendor/github.com/go-pkgz/rest/.gitignore similarity index 96% rename from vendor/github.com/go-pkgz/repeater/.gitignore rename to vendor/github.com/go-pkgz/rest/.gitignore index f1c181ec..cdc02295 100644 --- a/vendor/github.com/go-pkgz/repeater/.gitignore +++ b/vendor/github.com/go-pkgz/rest/.gitignore @@ -10,3 +10,4 @@ # Output of the go coverage tool, specifically when used with LiteIDE *.out +vendor \ No newline at end of file diff --git a/vendor/github.com/go-pkgz/notify/.golangci.yml b/vendor/github.com/go-pkgz/rest/.golangci.yml similarity index 74% rename from vendor/github.com/go-pkgz/notify/.golangci.yml rename to vendor/github.com/go-pkgz/rest/.golangci.yml index 3bb1e475..010de12f 100644 --- a/vendor/github.com/go-pkgz/notify/.golangci.yml +++ b/vendor/github.com/go-pkgz/rest/.golangci.yml @@ -1,15 +1,14 @@ -run: - timeout: 5m - output: - format: tab - skip-dirs: - - vendor - linters-settings: govet: check-shadowing: true + golint: + min-confidence: 0 + gocyclo: + min-complexity: 15 maligned: suggest-new: true + dupl: + threshold: 100 goconst: min-len: 2 min-occurrences: 2 @@ -24,10 +23,6 @@ linters-settings: - experimental disabled-checks: - wrapperFunc - - hugeParam - - rangeValCopy - - singleCaseSwitch - - ifElseChain linters: enable: @@ -41,6 +36,7 @@ linters: - dupl - misspell - unparam + - unused - typecheck - ineffassign - stylecheck @@ -50,17 +46,24 @@ linters: - nakedret - gosimple - prealloc - - whitespace fast: false disable-all: true +run: + output: + format: tab + skip-dirs: + - vendor + issues: exclude-rules: - - text: "at least one file in a package should have a package comment" + - text: 'Deferring unsafe method "Close" on type "io.ReadCloser"' linters: - - stylecheck + - gosec + - text: "should have a package comment, unless it's in another file for this package" + linters: + - golint - path: _test\.go linters: - - gosec - dupl exclude-use-default: false diff --git a/vendor/github.com/go-pkgz/repeater/LICENSE b/vendor/github.com/go-pkgz/rest/LICENSE similarity index 97% rename from vendor/github.com/go-pkgz/repeater/LICENSE rename to vendor/github.com/go-pkgz/rest/LICENSE index ac540250..ca125214 100644 --- a/vendor/github.com/go-pkgz/repeater/LICENSE +++ b/vendor/github.com/go-pkgz/rest/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2019 Umputun +Copyright (c) 2018 Umputun Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/vendor/github.com/go-pkgz/rest/README.md b/vendor/github.com/go-pkgz/rest/README.md new file mode 100644 index 00000000..dd1a2925 --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/README.md @@ -0,0 +1,244 @@ +## REST helpers and middleware [![Build Status](https://github.com/go-pkgz/rest/workflows/build/badge.svg)](https://github.com/go-pkgz/rest/actions) [![Go Report Card](https://goreportcard.com/badge/github.com/go-pkgz/rest)](https://goreportcard.com/report/github.com/go-pkgz/rest) [![Coverage Status](https://coveralls.io/repos/github/go-pkgz/rest/badge.svg?branch=master)](https://coveralls.io/github/go-pkgz/rest?branch=master) [![godoc](https://godoc.org/github.com/go-pkgz/rest?status.svg)](https://godoc.org/github.com/go-pkgz/rest) + + +## Install and update + +`go get -u github.com/go-pkgz/rest` + +## Middlewares + +### AppInfo middleware + +Adds info to every response header: +- App-Name - application name +- App-Version - application version +- Org - organization +- M-Host - host name from instance-level `$MHOST` env + +### Ping-Pong middleware + +Responds with `pong` on `GET /ping`. Also, responds to anything with `/ping` suffix, like `/v2/ping`. + +Example for both: + +``` +> http GET https://remark42.radio-t.com/ping + +HTTP/1.1 200 OK +Date: Sun, 15 Jul 2018 19:40:31 GMT +Content-Type: text/plain +Content-Length: 4 +Connection: keep-alive +App-Name: remark42 +App-Version: master-ed92a0b-20180630-15:59:56 +Org: Umputun + +pong +``` + +### Health middleware + +Responds with the status 200 if all health checks passed, 503 if any failed. Both health path and check functions passed by consumer. +For production usage this middleware should be used with throttler/limiter and, optionally, with some auth middlewares + +Example of usage: + +```go + check1 := func(ctx context.Context) (name string, err error) { + // do some check, for example check DB connection + return "check1", nil // all good, passed + } + check2 := func(ctx context.Context) (name string, err error) { + // do some other check, for example ping an external service + return "check2", errors.New("some error") // check failed + } + + router := chi.NewRouter() + router.Use(rest.Health("/health", check1, check2)) +``` + +example of the actual call and response: + +``` +> http GET https://example.com/health + +HTTP/1.1 503 Service Unavailable +Date: Sun, 15 Jul 2018 19:40:31 GMT +Content-Type: application/json; charset=utf-8 +Content-Length: 36 + +[ + {"name":"check1","status":"ok"}, + {"name":"check2","status":"failed","error":"some error"} +] +``` + +_this middleware is pretty basic, but can be used for simple health checks. For more complex cases, like async/cached health checks see [alexliesenfeld/health](https://github.com/alexliesenfeld/health)_ + +### Logger middleware + +Logs request, request handling time and response. Log record fields in order of occurrence: + +- Request's HTTP method +- Requested URL (with sanitized query) +- Remote IP +- Response's HTTP status code +- Response body size +- Request handling time +- Userinfo associated with the request (optional) +- Request subject (optional) +- Request ID (if `X-Request-ID` present) +- Request body (optional) + +_remote IP can be masked with user defined function_ + +example: `019/03/05 17:26:12.976 [INFO] GET - /api/v1/find?site=remark - 8e228e9cfece - 200 (115) - 4.47784618s` + +### Recoverer middleware + +Recoverer is a middleware that recovers from panics, logs the panic (and a backtrace), +and returns an HTTP 500 (Internal Server Error) status if possible. +It prevents server crashes in case of panic in one of the controllers. + +### OnlyFrom middleware + +OnlyFrom middleware allows access from a limited list of source IPs. +Such IPs can be defined as complete ip (like 192.168.1.12), prefix (129.168.) or CIDR (192.168.0.0/16). +The middleware will respond with `StatusForbidden` (403) if the request comes from a different IP. +It supports both IPv4 and IPv6 and checks the usual headers like `X-Forwarded-For` and `X-Real-IP` and the remote address. + +_Note: headers should be trusted and set by a proxy, otherwise it is possible to spoof them._ + +### Metrics middleware + +Metrics middleware responds to GET /metrics with list of [expvar](https://golang.org/pkg/expvar/). +Optionally allows a restricted list of source ips. + +### BlackWords middleware + +BlackWords middleware doesn't allow user-defined words in the request body. + +### SizeLimit middleware + +SizeLimit middleware checks if body size is above the limit and returns `StatusRequestEntityTooLarge` (413) + +### Trace middleware + +The `Trace` middleware is designed to add request tracing functionality. It looks for the `X-Request-ID` header in +the incoming HTTP request. If not found, a random ID is generated. This trace ID is then set in the response headers +and added to the request's context. + +### Deprecation middleware + +Adds the HTTP Deprecation response header, see [draft-dalal-deprecation-header-00](https://tools.ietf.org/id/draft-dalal-deprecation-header-00.html +) + +### BasicAuth middleware + +BasicAuth middleware requires basic auth and matches user & passwd with client-provided checker. In case if no basic auth headers returns +`StatusUnauthorized`, in case if checker failed - `StatusForbidden` + +### Rewrite middleware + +The `Rewrite` middleware is designed to rewrite the URL path based on a given rule, similar to how URL rewriting is done in nginx. It supports regular expressions for pattern matching and prevents multiple rewrites. + +For example, `Rewrite("^/sites/(.*)/settings/$", "/sites/settings/$1")` will change request's URL from `/sites/id1/settings/` to `/sites/settings/id1` + +### NoCache middleware + +Sets a number of HTTP headers to prevent a router (handler's) response from being cached by an upstream proxy and/or client. + +### Headers middleware + +Sets headers (passed as key:value) to requests. I.e. `rest.Headers("Server:MyServer", "X-Blah:Foo")` + +### Gzip middleware + +Compresses response with gzip. + +### RealIP middleware + +RealIP is a middleware that sets a http.Request's RemoteAddr to the results of parsing either the X-Forwarded-For or X-Real-IP headers. + +### Maybe middleware + +Maybe middleware allows changing the flow of the middleware stack execution depending on the return +value of maybeFn(request). This is useful, for example, to skip a middleware handler if a request does not satisfy the maybeFn logic. + +### Reject middleware + +Reject is a middleware that rejects requests with a given status code and message based on a user-defined function. +This is useful, for example, to reject requests to a particular resource based on a request header, +or to implement a conditional request handler based on service parameters. + +example with chi router: + +```go + router := chi.NewRouter() + + rejectFn := func(r *http.Request) (bool) { + return r.Header.Get("X-Request-Id") == "" // reject if no X-Request-Id header + } + + router.Use(rest.Reject(http.StatusBadRequest, "X-Request-Id header is required", rejectFn)) +``` + +### Benchmarks middleware + +Benchmarks middleware allows measuring the time of request handling, number of requests per second and report aggregated metrics. +This middleware keeps track of the request in the memory and keep up to 900 points (15 minutes, data-point per second). + +To retrieve the data user should call `Stats(d duration)` method. +The `duration` is the time window for which the benchmark data should be returned. +It can be any duration from 1s to 15m. Note: all the time data is in microseconds. + +example with chi router: + +```go + router := chi.NewRouter() + bench = rest.NewBenchmarks() + router.Use(bench.Middleware) + ... + router.Get("/bench", func(w http.ResponseWriter, r *http.Request) { + resp := struct { + OneMin rest.BenchmarkStats `json:"1min"` + FiveMin rest.BenchmarkStats `json:"5min"` + FifteenMin rest.BenchmarkStats `json:"15min"` + }{ + bench.Stats(time.Minute), + bench.Stats(time.Minute * 5), + bench.Stats(time.Minute * 15), + } + render.JSON(w, r, resp) + }) +``` + +## Helpers + +- `rest.Wrap` - converts a list of middlewares to nested handlers calls (in reverse order) +- `rest.JSON` - map alias, just for convenience `type JSON map[string]interface{}` +- `rest.RenderJSON` - renders json response from `interface{}` +- `rest.RenderJSONFromBytes` - renders json response from `[]byte` +- `rest.RenderJSONWithHTML` - renders json response with html tags and forced `charset=utf-8` +- `rest.SendErrorJSON` - makes `{error: blah, details: blah}` json body and responds with given error code. Also, adds context to the logged message +- `rest.NewErrorLogger` - creates a struct providing shorter form of logger call +- `rest.FileServer` - creates a file server for static assets with directory listing disabled +- `realip.Get` - returns client's IP address +- `rest.ParseFromTo` - parses "from" and "to" request's query params with various formats + +## Profiler + +Profiler is a convenient sub-router used for mounting net/http/pprof, i.e. + +```go + func MyService() http.Handler { + r := chi.NewRouter() + // ..middlewares + r.Mount("/debug", middleware.Profiler()) + // ..routes + return r + } +``` + +It exposes a bunch of `/pprof/*` endpoints as well as `/vars`. Builtin support for `onlyIps` allows restricting access, which is important if it runs on a publicly exposed port. However, counting on IP check only is not that reliable way to limit request and for production use it would be better to add some sort of auth (for example provided `BasicAuth` middleware) or run with a separate http server, exposed to internal ip/port only. + diff --git a/vendor/github.com/go-pkgz/rest/basic_auth.go b/vendor/github.com/go-pkgz/rest/basic_auth.go new file mode 100644 index 00000000..d184abb1 --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/basic_auth.go @@ -0,0 +1,73 @@ +package rest + +import ( + "context" + "crypto/subtle" + "net/http" +) + +const baContextKey = "authorizedWithBasicAuth" + +// BasicAuth middleware requires basic auth and matches user & passwd with client-provided checker +func BasicAuth(checker func(user, passwd string) bool) func(http.Handler) http.Handler { + + return func(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + + u, p, ok := r.BasicAuth() + if !ok { + w.WriteHeader(http.StatusUnauthorized) + return + } + if !checker(u, p) { + w.WriteHeader(http.StatusForbidden) + return + } + h.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), contextKey(baContextKey), true))) + } + return http.HandlerFunc(fn) + } +} + +// BasicAuthWithUserPasswd middleware requires basic auth and matches user & passwd with client-provided values +func BasicAuthWithUserPasswd(user, passwd string) func(http.Handler) http.Handler { + checkFn := func(reqUser, reqPasswd string) bool { + matchUser := subtle.ConstantTimeCompare([]byte(user), []byte(reqUser)) + matchPass := subtle.ConstantTimeCompare([]byte(passwd), []byte(reqPasswd)) + return matchUser == 1 && matchPass == 1 + } + return BasicAuth(checkFn) +} + +// IsAuthorized returns true is user authorized. +// it can be used in handlers to check if BasicAuth middleware was applied +func IsAuthorized(ctx context.Context) bool { + v := ctx.Value(contextKey(baContextKey)) + return v != nil && v.(bool) +} + +// BasicAuthWithPrompt middleware requires basic auth and matches user & passwd with client-provided values +// If the user is not authorized, it will prompt for basic auth +func BasicAuthWithPrompt(user, passwd string) func(http.Handler) http.Handler { + checkFn := func(reqUser, reqPasswd string) bool { + matchUser := subtle.ConstantTimeCompare([]byte(user), []byte(reqUser)) + matchPass := subtle.ConstantTimeCompare([]byte(passwd), []byte(reqPasswd)) + return matchUser == 1 && matchPass == 1 + } + + return func(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + + // extract basic auth from request + u, p, ok := r.BasicAuth() + if ok && checkFn(u, p) { + h.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), contextKey(baContextKey), true))) + return + } + // not authorized, prompt for basic auth + w.Header().Set("WWW-Authenticate", `Basic realm="restricted", charset="UTF-8"`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + } + return http.HandlerFunc(fn) + } +} diff --git a/vendor/github.com/go-pkgz/rest/benchmarks.go b/vendor/github.com/go-pkgz/rest/benchmarks.go new file mode 100644 index 00000000..702e517b --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/benchmarks.go @@ -0,0 +1,159 @@ +package rest + +import ( + "container/list" + "net/http" + "sync" + "time" +) + +var maxTimeRangeDefault = time.Duration(15) * time.Minute + +// Benchmarks is a basic benchmarking middleware collecting and reporting performance metrics +// It keeps track of the requests speeds and counts in 1s benchData buckets ,limiting the number of buckets +// to maxTimeRange. User can request the benchmark for any time duration. This is intended to be used +// for retrieving the benchmark data for the last minute, 5 minutes and up to maxTimeRange. +type Benchmarks struct { + st time.Time + data *list.List + lock sync.RWMutex + maxTimeRange time.Duration + + nowFn func() time.Time // for testing only +} + +type benchData struct { + // 1s aggregates + requests int + respTime time.Duration + minRespTime time.Duration + maxRespTime time.Duration + ts time.Time +} + +// BenchmarkStats holds the stats for a given interval +type BenchmarkStats struct { + Requests int `json:"requests"` + RequestsSec float64 `json:"requests_sec"` + AverageRespTime int64 `json:"average_resp_time"` + MinRespTime int64 `json:"min_resp_time"` + MaxRespTime int64 `json:"max_resp_time"` +} + +// NewBenchmarks creates a new benchmark middleware +func NewBenchmarks() *Benchmarks { + res := &Benchmarks{ + st: time.Now(), + data: list.New(), + nowFn: time.Now, + maxTimeRange: maxTimeRangeDefault, + } + return res +} + +// WithTimeRange sets the maximum time range for the benchmark to keep data for. +// Default is 15 minutes. The increase of this range will change memory utilization as each second of the range +// kept as benchData aggregate. The default means 15*60 = 900 seconds of data aggregate. +// Larger range allows for longer time periods to be benchmarked. +func (b *Benchmarks) WithTimeRange(max time.Duration) *Benchmarks { + b.lock.Lock() + defer b.lock.Unlock() + b.maxTimeRange = max + return b +} + +// Handler calculates 1/5/10m request per second and allows to access those values +func (b *Benchmarks) Handler(next http.Handler) http.Handler { + + fn := func(w http.ResponseWriter, r *http.Request) { + st := b.nowFn() + defer func() { + b.update(time.Since(st)) + }() + next.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) +} + +func (b *Benchmarks) update(reqDuration time.Duration) { + now := b.nowFn().Truncate(time.Second) + + b.lock.Lock() + defer b.lock.Unlock() + + // keep maxTimeRange in the list, drop the rest + for e := b.data.Front(); e != nil; e = e.Next() { + if b.data.Front().Value.(benchData).ts.After(b.nowFn().Add(-b.maxTimeRange)) { + break + } + b.data.Remove(b.data.Front()) + } + + last := b.data.Back() + if last == nil || last.Value.(benchData).ts.Before(now) { + b.data.PushBack(benchData{requests: 1, respTime: reqDuration, ts: now, + minRespTime: reqDuration, maxRespTime: reqDuration}) + return + } + + bd := last.Value.(benchData) + bd.requests++ + bd.respTime += reqDuration + + if bd.minRespTime == 0 || reqDuration < bd.minRespTime { + bd.minRespTime = reqDuration + } + if bd.maxRespTime == 0 || reqDuration > bd.maxRespTime { + bd.maxRespTime = reqDuration + } + + last.Value = bd +} + +// Stats returns the current benchmark stats for the given duration +func (b *Benchmarks) Stats(interval time.Duration) BenchmarkStats { + if interval < time.Second { // minimum interval is 1s due to the bucket size + return BenchmarkStats{} + } + + b.lock.RLock() + defer b.lock.RUnlock() + + var ( + requests int + respTime time.Duration + ) + + stInterval, fnInterval := time.Time{}, time.Time{} + var minRespTime, maxRespTime time.Duration + for e := b.data.Back(); e != nil; e = e.Prev() { // reverse order + bd := e.Value.(benchData) + if bd.ts.Before(b.nowFn().Add(-interval)) { + break + } + if minRespTime == 0 || bd.minRespTime < minRespTime { + minRespTime = bd.minRespTime + } + if maxRespTime == 0 || bd.maxRespTime > maxRespTime { + maxRespTime = bd.maxRespTime + } + requests += bd.requests + respTime += bd.respTime + if fnInterval.IsZero() { + fnInterval = bd.ts.Add(time.Second) + } + stInterval = bd.ts + } + + if requests == 0 { + return BenchmarkStats{} + } + + return BenchmarkStats{ + Requests: requests, + RequestsSec: float64(requests) / (fnInterval.Sub(stInterval).Seconds()), + AverageRespTime: respTime.Microseconds() / int64(requests), + MinRespTime: minRespTime.Microseconds(), + MaxRespTime: maxRespTime.Microseconds(), + } +} diff --git a/vendor/github.com/go-pkgz/rest/blackwords.go b/vendor/github.com/go-pkgz/rest/blackwords.go new file mode 100644 index 00000000..dd120d7a --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/blackwords.go @@ -0,0 +1,39 @@ +package rest + +import ( + "bytes" + "io" + "net/http" + "strings" +) + +// BlackWords middleware doesn't allow some words in the request body +func BlackWords(words ...string) func(http.Handler) http.Handler { + + return func(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + + if content, err := io.ReadAll(r.Body); err == nil { + body := strings.ToLower(string(content)) + r.Body = io.NopCloser(bytes.NewReader(content)) + + if len(body) > 0 { + for _, word := range words { + if strings.Contains(body, strings.ToLower(word)) { + w.WriteHeader(http.StatusForbidden) + RenderJSON(w, JSON{"error": "one of blacklisted words detected"}) + return + } + } + } + } + h.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) + } +} + +// BlackWordsFn middleware uses func to get the list and doesn't allow some words in the request body +func BlackWordsFn(fn func() []string) func(http.Handler) http.Handler { + return BlackWords(fn()...) +} diff --git a/vendor/github.com/go-pkgz/rest/cache_control.go b/vendor/github.com/go-pkgz/rest/cache_control.go new file mode 100644 index 00000000..b5950557 --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/cache_control.go @@ -0,0 +1,62 @@ +package rest + +import ( + "crypto/sha1" //nolint not used for cryptography + "fmt" + "net/http" + "strings" + "time" +) + +// CacheControl is a middleware setting cache expiration. Using url+version for etag +func CacheControl(expiration time.Duration, version string) func(http.Handler) http.Handler { + + etag := func(r *http.Request, version string) string { + s := fmt.Sprintf("%s:%s", version, r.URL.String()) + return fmt.Sprintf("%x", sha1.Sum([]byte(s))) //nolint + } + + return func(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + e := `"` + etag(r, version) + `"` + w.Header().Set("Etag", e) + w.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d, no-cache", int(expiration.Seconds()))) + + if match := r.Header.Get("If-None-Match"); match != "" { + if strings.Contains(match, e) { + w.WriteHeader(http.StatusNotModified) + return + } + } + h.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) + } +} + +// CacheControlDynamic is a middleware setting cache expiration. Using url+ func(r) for etag +func CacheControlDynamic(expiration time.Duration, versionFn func(r *http.Request) string) func(http.Handler) http.Handler { + + etag := func(r *http.Request, version string) string { + s := fmt.Sprintf("%s:%s", version, r.URL.String()) + return fmt.Sprintf("%x", sha1.Sum([]byte(s))) //nolint + } + + return func(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + + e := `"` + etag(r, versionFn(r)) + `"` + w.Header().Set("Etag", e) + w.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d, no-cache", int(expiration.Seconds()))) + + if match := r.Header.Get("If-None-Match"); match != "" { + if strings.Contains(match, e) { + w.WriteHeader(http.StatusNotModified) + return + } + } + h.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) + } +} diff --git a/vendor/github.com/go-pkgz/rest/depricattion.go b/vendor/github.com/go-pkgz/rest/depricattion.go new file mode 100644 index 00000000..ded640b6 --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/depricattion.go @@ -0,0 +1,21 @@ +package rest + +import ( + "fmt" + "net/http" + "time" +) + +// Deprecation adds a header 'Deprecation: version="version", date="date" header' +// see https://tools.ietf.org/id/draft-dalal-deprecation-header-00.html +func Deprecation(version string, date time.Time) func(http.Handler) http.Handler { + f := func(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + headerVal := fmt.Sprintf("version=%q, date=%q", version, date.Format(time.RFC3339)) + w.Header().Set("Deprecation", headerVal) + h.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) + } + return f +} diff --git a/vendor/github.com/go-pkgz/rest/file_server.go b/vendor/github.com/go-pkgz/rest/file_server.go new file mode 100644 index 00000000..ccf47df5 --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/file_server.go @@ -0,0 +1,185 @@ +package rest + +import ( + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" +) + +// FS provides http.FileServer handler to serve static files from a http.FileSystem, +// prevents directory listing by default and supports spa-friendly mode (off by default) returning /index.html on 404. +// - public defines base path of the url, i.e. for http://example.com/static/* it should be /static +// - local for the local path to the root of the served directory +// - notFound is the reader for the custom 404 html, can be nil for default +type FS struct { + public, root string + notFound io.Reader + isSpa bool + enableListing bool + handler http.HandlerFunc +} + +// NewFileServer creates file server with optional spa mode and optional direcroty listing (disabled by default) +func NewFileServer(public, local string, options ...FsOpt) (*FS, error) { + res := FS{ + public: public, + notFound: nil, + isSpa: false, + enableListing: false, + } + + root, err := filepath.Abs(local) + if err != nil { + return nil, fmt.Errorf("can't get absolute path for %s: %w", local, err) + } + res.root = root + + if _, err = os.Stat(root); os.IsNotExist(err) { + return nil, fmt.Errorf("local path %s doesn't exist: %w", root, err) + } + + for _, opt := range options { + err = opt(&res) + if err != nil { + return nil, err + } + } + + cfs := customFS{ + fs: http.Dir(root), + spa: res.isSpa, + listing: res.enableListing, + } + f := http.StripPrefix(public, http.FileServer(cfs)) + res.handler = func(w http.ResponseWriter, r *http.Request) { f.ServeHTTP(w, r) } + + if !res.enableListing { + h, err := custom404Handler(f, res.notFound) + if err != nil { + return nil, err + } + res.handler = func(w http.ResponseWriter, r *http.Request) { h.ServeHTTP(w, r) } + } + + return &res, nil +} + +// FileServer is a shortcut for making FS with listing disabled and the custom noFound reader (can be nil). +// Deprecated: the method is for back-compatibility only and user should use the universal NewFileServer instead +func FileServer(public, local string, notFound io.Reader) (http.Handler, error) { + return NewFileServer(public, local, FsOptCustom404(notFound)) +} + +// FileServerSPA is a shortcut for making FS with SPA-friendly handling of 404, listing disabled and the custom noFound reader (can be nil). +// Deprecated: the method is for back-compatibility only and user should use the universal NewFileServer instead +func FileServerSPA(public, local string, notFound io.Reader) (http.Handler, error) { + return NewFileServer(public, local, FsOptCustom404(notFound), FsOptSPA) +} + +// ServeHTTP makes FileServer compatible with http.Handler interface +func (fs *FS) ServeHTTP(w http.ResponseWriter, r *http.Request) { + fs.handler(w, r) +} + +// FsOpt defines functional option type +type FsOpt func(fs *FS) error + +// FsOptSPA turns on SPA mode returning "/index.html" on not-found +func FsOptSPA(fs *FS) error { + fs.isSpa = true + return nil +} + +// FsOptListing turns on directory listing +func FsOptListing(fs *FS) error { + fs.enableListing = true + return nil +} + +// FsOptCustom404 sets custom 404 reader +func FsOptCustom404(fr io.Reader) FsOpt { + return func(fs *FS) error { + fs.notFound = fr + return nil + } +} + +// customFS wraps http.FileSystem with spa and no-listing optional support +type customFS struct { + fs http.FileSystem + spa bool + listing bool +} + +// Open file on FS, for directory enforce index.html and fail on a missing index +func (cfs customFS) Open(name string) (http.File, error) { + + f, err := cfs.fs.Open(name) + if err != nil { + if cfs.spa { + return cfs.fs.Open("/index.html") + } + return nil, err + } + + finfo, err := f.Stat() + if err != nil { + return nil, err + } + + if finfo.IsDir() { + index := strings.TrimSuffix(name, "/") + "/index.html" + if _, err := cfs.fs.Open(index); err == nil { // index.html will be served if found + return f, nil + } + // no index.html in directory + if !cfs.listing { // listing disabled + if _, err := cfs.fs.Open(index); err != nil { + return nil, err + } + } + } + + return f, nil +} + +// respWriter404 intercept Write to provide custom 404 response +type respWriter404 struct { + http.ResponseWriter + status int + msg []byte +} + +func (w *respWriter404) WriteHeader(status int) { + w.status = status + if status == http.StatusNotFound { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + } + w.ResponseWriter.WriteHeader(status) +} + +func (w *respWriter404) Write(p []byte) (n int, err error) { + if w.status != http.StatusNotFound || w.msg == nil { + return w.ResponseWriter.Write(p) + } + _, err = w.ResponseWriter.Write(w.msg) + return len(p), err +} + +func custom404Handler(next http.Handler, notFound io.Reader) (http.Handler, error) { + if notFound == nil { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { next.ServeHTTP(w, r) }), nil + } + + body, err := io.ReadAll(notFound) + if err != nil { + return nil, err + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(&respWriter404{ResponseWriter: w, msg: body}, r) + }), nil +} diff --git a/vendor/github.com/go-pkgz/rest/gzip.go b/vendor/github.com/go-pkgz/rest/gzip.go new file mode 100644 index 00000000..a7328b05 --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/gzip.go @@ -0,0 +1,88 @@ +package rest + +import ( + "compress/gzip" + "io" + "net/http" + "strings" + "sync" +) + +var gzDefaultContentTypes = []string{ + "text/css", + "text/javascript", + "text/xml", + "text/html", + "text/plain", + "application/javascript", + "application/x-javascript", + "application/json", +} + +var gzPool = sync.Pool{ + New: func() interface{} { return gzip.NewWriter(io.Discard) }, +} + +type gzipResponseWriter struct { + io.Writer + http.ResponseWriter +} + +func (w *gzipResponseWriter) WriteHeader(status int) { + w.Header().Del("Content-Length") + w.ResponseWriter.WriteHeader(status) +} + +func (w *gzipResponseWriter) Write(b []byte) (int, error) { + return w.Writer.Write(b) +} + +// Gzip is a middleware compressing response +func Gzip(contentTypes ...string) func(http.Handler) http.Handler { + + gzCts := gzDefaultContentTypes + if len(contentTypes) > 0 { + gzCts = contentTypes + } + + contentType := func(r *http.Request) string { + result := r.Header.Get("Content-type") + if result == "" { + return "application/octet-stream" + } + return result + } + + f := func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + next.ServeHTTP(w, r) + return + } + + var gzOk bool + ctype := contentType(r) + for _, c := range gzCts { + if strings.HasPrefix(strings.ToLower(ctype), strings.ToLower(c)) { + gzOk = true + break + } + } + + if !gzOk { + next.ServeHTTP(w, r) + return + } + + w.Header().Set("Content-Encoding", "gzip") + gz := gzPool.Get().(*gzip.Writer) + defer gzPool.Put(gz) + + gz.Reset(w) + defer gz.Close() + + next.ServeHTTP(&gzipResponseWriter{ResponseWriter: w, Writer: gz}, r) + }) + } + return f +} diff --git a/vendor/github.com/go-pkgz/rest/httperrors.go b/vendor/github.com/go-pkgz/rest/httperrors.go new file mode 100644 index 00000000..704ef745 --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/httperrors.go @@ -0,0 +1,67 @@ +package rest + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "runtime" + "strings" + + "github.com/go-pkgz/rest/logger" +) + +// ErrorLogger wraps logger.Backend +type ErrorLogger struct { + l logger.Backend +} + +// NewErrorLogger creates ErrorLogger for given Backend +func NewErrorLogger(l logger.Backend) *ErrorLogger { + return &ErrorLogger{l: l} +} + +// Log sends json error message {error: msg} with error code and logging error and caller +func (e *ErrorLogger) Log(w http.ResponseWriter, r *http.Request, httpCode int, err error, msg ...string) { + m := "" + if len(msg) > 0 { + m = strings.Join(msg, ". ") + } + if e.l != nil { + e.l.Logf("%s", errDetailsMsg(r, httpCode, err, m)) + } + renderJSONWithStatus(w, JSON{"error": m}, httpCode) +} + +// SendErrorJSON sends {error: msg} with error code and logging error and caller +func SendErrorJSON(w http.ResponseWriter, r *http.Request, l logger.Backend, code int, err error, msg string) { + if l != nil { + l.Logf("%s", errDetailsMsg(r, code, err, msg)) + } + renderJSONWithStatus(w, JSON{"error": msg}, code) +} + +func errDetailsMsg(r *http.Request, code int, err error, msg string) string { + + q := r.URL.String() + if qun, e := url.QueryUnescape(q); e == nil { + q = qun + } + + srcFileInfo := "" + if pc, file, line, ok := runtime.Caller(2); ok { + fnameElems := strings.Split(file, "/") + funcNameElems := strings.Split(runtime.FuncForPC(pc).Name(), "/") + srcFileInfo = fmt.Sprintf(" [caused by %s:%d %s]", strings.Join(fnameElems[len(fnameElems)-3:], "/"), + line, funcNameElems[len(funcNameElems)-1]) + } + + remoteIP := r.RemoteAddr + if pos := strings.Index(remoteIP, ":"); pos >= 0 { + remoteIP = remoteIP[:pos] + } + if err == nil { + err = errors.New("no error") + } + return fmt.Sprintf("%s - %v - %d - %s - %s%s", msg, err, code, remoteIP, q, srcFileInfo) +} diff --git a/vendor/github.com/go-pkgz/rest/logger/logger.go b/vendor/github.com/go-pkgz/rest/logger/logger.go new file mode 100644 index 00000000..dafdd3bc --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/logger/logger.go @@ -0,0 +1,359 @@ +// Package logger implements logging middleware +package logger + +import ( + "bufio" + "bytes" + "fmt" + "io" + "log" + "net" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + "github.com/go-pkgz/rest/realip" +) + +// Middleware is a logger for rest requests. +type Middleware struct { + prefix string + logBody bool + maxBodySize int + ipFn func(ip string) string + userFn func(r *http.Request) (string, error) + subjFn func(r *http.Request) (string, error) + log Backend + apacheCombined bool +} + +// Backend is logging backend +type Backend interface { + Logf(format string, args ...interface{}) +} + +type logParts struct { + duration time.Duration + rawURL string + method string + remoteIP string + statusCode int + respSize int + host string + + prefix string + user string + body string +} + +type stdBackend struct{} + +func (s stdBackend) Logf(format string, args ...interface{}) { + log.Printf(format, args...) +} + +// Logger is a default logger middleware with "REST" prefix +func Logger(next http.Handler) http.Handler { + l := New(Prefix("REST")) + return l.Handler(next) + +} + +// New makes rest logger with given options +func New(options ...Option) *Middleware { + res := Middleware{ + prefix: "", + maxBodySize: 1024, + log: stdBackend{}, + } + for _, opt := range options { + opt(&res) + } + return &res +} + +// Handler middleware prints http log +func (l *Middleware) Handler(next http.Handler) http.Handler { + + formater := l.formatDefault + if l.apacheCombined { + formater = l.formatApacheCombined + } + + fn := func(w http.ResponseWriter, r *http.Request) { + ww := newCustomResponseWriter(w) + + user := "" + if l.userFn != nil { + if u, err := l.userFn(r); err == nil { + user = u + } + } + + body := l.getBody(r) + t1 := time.Now() + defer func() { + t2 := time.Now() + + u := *r.URL // shallow copy + u.RawQuery = l.sanitizeQuery(u.RawQuery) + rawurl := u.String() + if unescURL, err := url.QueryUnescape(rawurl); err == nil { + rawurl = unescURL + } + + remoteIP, err := realip.Get(r) + if err != nil { + remoteIP = "unknown ip" + } + if l.ipFn != nil { // mask ip with ipFn + remoteIP = l.ipFn(remoteIP) + } + + server := r.URL.Hostname() + if server == "" { + server = strings.Split(r.Host, ":")[0] + } + + p := &logParts{ + duration: t2.Sub(t1), + rawURL: rawurl, + method: r.Method, + host: server, + remoteIP: remoteIP, + statusCode: ww.status, + respSize: ww.size, + prefix: l.prefix, + user: user, + body: body, + } + + l.log.Logf(formater(r, p)) + }() + + next.ServeHTTP(ww, r) + } + return http.HandlerFunc(fn) +} + +func (l *Middleware) formatDefault(r *http.Request, p *logParts) string { + var bld strings.Builder + if l.prefix != "" { + _, _ = bld.WriteString(l.prefix) + _, _ = bld.WriteString(" ") + } + + _, _ = bld.WriteString(fmt.Sprintf("%s - %s - %s - %s - %d (%d) - %v", + p.method, p.rawURL, p.host, p.remoteIP, p.statusCode, p.respSize, p.duration)) + + if p.user != "" { + _, _ = bld.WriteString(" - ") + _, _ = bld.WriteString(p.user) + } + + if l.subjFn != nil { + if subj, err := l.subjFn(r); err == nil { + _, _ = bld.WriteString(" - ") + _, _ = bld.WriteString(subj) + } + } + + if traceID := r.Header.Get("X-Request-ID"); traceID != "" { + _, _ = bld.WriteString(" - ") + _, _ = bld.WriteString(traceID) + } + + if p.body != "" { + _, _ = bld.WriteString(" - ") + _, _ = bld.WriteString(p.body) + } + return bld.String() +} + +// 127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)" +// nolint gosec +func (l *Middleware) formatApacheCombined(r *http.Request, p *logParts) string { + username := "-" + if p.user != "" { + username = p.user + } + + var bld strings.Builder + bld.WriteString(p.remoteIP) + bld.WriteString(" - ") + bld.WriteString(username) + bld.WriteString(" [") + bld.WriteString(time.Now().Format("02/Jan/2006:15:04:05 -0700")) + bld.WriteString(`] "`) + bld.WriteString(p.method) + bld.WriteString(" ") + bld.WriteString(p.rawURL) + bld.WriteString(`" `) + bld.WriteString(r.Proto) + bld.WriteString(`" `) + bld.WriteString(strconv.Itoa(p.statusCode)) + bld.WriteString(" ") + bld.WriteString(strconv.Itoa(p.respSize)) + + bld.WriteString(` "`) + bld.WriteString(r.Header.Get("Referer")) + bld.WriteString(`" "`) + bld.WriteString(r.Header.Get("User-Agent")) + bld.WriteString(`"`) + return bld.String() +} + +var reMultWhtsp = regexp.MustCompile(`[\s\p{Zs}]{2,}`) + +func (l *Middleware) getBody(r *http.Request) string { + if !l.logBody { + return "" + } + + reader, body, hasMore, err := peek(r.Body, int64(l.maxBodySize)) + if err != nil { + return "" + } + + // "The Server will close the request body. The ServeHTTP Handler does not need to." + // https://golang.org/pkg/net/http/#Request + // So we can use ioutil.NopCloser() to make io.ReadCloser. + // Note that below assignment is not approved by the docs: + // "Except for reading the body, handlers should not modify the provided Request." + // https://golang.org/pkg/net/http/#Handler + r.Body = io.NopCloser(reader) + + if len(body) > 0 { + body = strings.Replace(body, "\n", " ", -1) + body = reMultWhtsp.ReplaceAllString(body, " ") + } + + if hasMore { + body += "..." + } + + return body +} + +// peek the first n bytes as string +func peek(r io.Reader, n int64) (reader io.Reader, s string, hasMore bool, err error) { + if n < 0 { + n = 0 + } + + buf := new(bytes.Buffer) + _, err = io.CopyN(buf, r, n+1) + if err == io.EOF { + str := buf.String() + return buf, str, false, nil + } + if err != nil { + return r, "", false, err + } + + // one extra byte is successfully read + s = buf.String() + s = s[:len(s)-1] + + return io.MultiReader(buf, r), s, true, nil +} + +var keysToHide = []string{"password", "passwd", "secret", "credentials", "token"} + +// Hide query values for keysToHide. May change order of query params. +// May escape unescaped query params. +func (l *Middleware) sanitizeQuery(rawQuery string) string { + // note that we skip non-nil error further + query, err := url.ParseQuery(rawQuery) + + isHidden := func(key string) bool { + for _, k := range keysToHide { + if strings.EqualFold(k, key) { + return true + } + } + return false + } + + present := false + for key, values := range query { + if isHidden(key) { + present = true + for i := range values { + values[i] = "********" + } + } + } + + // short circuit + if (err == nil) && !present { + return rawQuery + } + + return query.Encode() +} + +// AnonymizeIP is a function to reset the last part of IPv4 to 0. +// from 123.212.12.78 it will make 123.212.12.0 +func AnonymizeIP(ip string) string { + if ip == "" { + return "" + } + + parts := strings.Split(ip, ".") + if len(parts) != 4 { + return ip + } + + return strings.Join(parts[:3], ".") + ".0" +} + +// customResponseWriter is an HTTP response logger that keeps HTTP status code and +// the number of bytes written. +// It implements http.ResponseWriter, http.Flusher and http.Hijacker. +// Note that type assertion from http.ResponseWriter(customResponseWriter) to +// http.Flusher and http.Hijacker is always succeed but underlying http.ResponseWriter +// may not implement them. +type customResponseWriter struct { + http.ResponseWriter + status int + size int +} + +func newCustomResponseWriter(w http.ResponseWriter) *customResponseWriter { + return &customResponseWriter{ + ResponseWriter: w, + status: 200, + } +} + +// WriteHeader implements http.ResponseWriter and saves status +func (c *customResponseWriter) WriteHeader(status int) { + c.status = status + c.ResponseWriter.WriteHeader(status) +} + +// Write implements http.ResponseWriter and tracks number of bytes written +func (c *customResponseWriter) Write(b []byte) (int, error) { + size, err := c.ResponseWriter.Write(b) + c.size += size + return size, err +} + +// Flush implements http.Flusher +func (c *customResponseWriter) Flush() { + if f, ok := c.ResponseWriter.(http.Flusher); ok { + f.Flush() + } +} + +// Hijack implements http.Hijacker +func (c *customResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + if hj, ok := c.ResponseWriter.(http.Hijacker); ok { + return hj.Hijack() + } + return nil, nil, fmt.Errorf("ResponseWriter does not implement the Hijacker interface") //nolint:golint //capital letter is OK here +} diff --git a/vendor/github.com/go-pkgz/rest/logger/options.go b/vendor/github.com/go-pkgz/rest/logger/options.go new file mode 100644 index 00000000..24f99de9 --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/logger/options.go @@ -0,0 +1,63 @@ +package logger + +import ( + "net/http" +) + +// Option func type +type Option func(l *Middleware) + +// WithBody triggers request body logging. Body size is limited (default 1k) +func WithBody(l *Middleware) { + l.logBody = true +} + +// MaxBodySize sets size of the logged part of the request body. +func MaxBodySize(max int) Option { + return func(l *Middleware) { + if max >= 0 { + l.maxBodySize = max + } + } +} + +// Prefix sets log line prefix. +func Prefix(prefix string) Option { + return func(l *Middleware) { + l.prefix = prefix + } +} + +// IPfn sets IP masking function. If ipFn is nil then IP address will be logged as is. +func IPfn(ipFn func(ip string) string) Option { + return func(l *Middleware) { + l.ipFn = ipFn + } +} + +// UserFn triggers user name logging if userFn is not nil. +func UserFn(userFn func(r *http.Request) (string, error)) Option { + return func(l *Middleware) { + l.userFn = userFn + } +} + +// SubjFn triggers subject logging if subjFn is not nil. +func SubjFn(subjFn func(r *http.Request) (string, error)) Option { + return func(l *Middleware) { + l.subjFn = subjFn + } +} + +// ApacheCombined sets format to Apache Combined Log. +// See http://httpd.apache.org/docs/2.2/logs.html#combined +func ApacheCombined(l *Middleware) { + l.apacheCombined = true +} + +// Log sets logging backend. +func Log(log Backend) Option { + return func(l *Middleware) { + l.log = log + } +} diff --git a/vendor/github.com/go-pkgz/rest/metrics.go b/vendor/github.com/go-pkgz/rest/metrics.go new file mode 100644 index 00000000..1cf0a419 --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/metrics.go @@ -0,0 +1,28 @@ +package rest + +import ( + "expvar" + "fmt" + "net/http" + "strings" +) + +// Metrics responds to GET /metrics with list of expvar +func Metrics(onlyIps ...string) func(http.Handler) http.Handler { + return func(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + if r.Method == "GET" && strings.HasSuffix(strings.ToLower(r.URL.Path), "/metrics") { + if matched, ip, err := matchSourceIP(r, onlyIps); !matched || err != nil { + w.WriteHeader(http.StatusForbidden) + RenderJSON(w, JSON{"error": fmt.Sprintf("ip %s rejected", ip)}) + return + } + expvar.Handler().ServeHTTP(w, r) + return + } + h.ServeHTTP(w, r) + } + + return http.HandlerFunc(fn) + } +} diff --git a/vendor/github.com/go-pkgz/rest/middleware.go b/vendor/github.com/go-pkgz/rest/middleware.go new file mode 100644 index 00000000..6e503379 --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/middleware.go @@ -0,0 +1,184 @@ +package rest + +import ( + "context" + "net/http" + "os" + "runtime/debug" + "strings" + + "github.com/go-pkgz/rest/logger" + "github.com/go-pkgz/rest/realip" +) + +// Wrap converts a list of middlewares to nested calls (in reverse order) +func Wrap(handler http.Handler, mws ...func(http.Handler) http.Handler) http.Handler { + if len(mws) == 0 { + return handler + } + res := handler + for i := len(mws) - 1; i >= 0; i-- { + res = mws[i](res) + } + return res +} + +// AppInfo adds custom app-info to the response header +func AppInfo(app, author, version string) func(http.Handler) http.Handler { + f := func(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Author", author) + w.Header().Set("App-Name", app) + w.Header().Set("App-Version", version) + if mhost := os.Getenv("MHOST"); mhost != "" { + w.Header().Set("Host", mhost) + } + h.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) + } + return f +} + +// Ping middleware response with pong to /ping. Stops chain if ping request detected +func Ping(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + + if r.Method == "GET" && strings.HasSuffix(strings.ToLower(r.URL.Path), "/ping") { + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("pong")) + return + } + next.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) +} + +// Health middleware response with health info and status (200 if healthy). Stops chain if health request detected +// passed checkers implements custom health checks and returns error if health check failed. The check has to return name +// regardless to the error state. +// For production usage this middleware should be used with throttler and, optionally, with BasicAuth middlewares +func Health(path string, checkers ...func(ctx context.Context) (name string, err error)) func(http.Handler) http.Handler { + + type hr struct { + Name string `json:"name"` + Status string `json:"status"` + Error string `json:"error,omitempty"` + } + + return func(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" || !strings.EqualFold(r.URL.Path, path) { + h.ServeHTTP(w, r) // not the health check request, continue the chain + return + } + resp := []hr{} + var anyError bool + for _, check := range checkers { + name, err := check(r.Context()) + hh := hr{Name: name, Status: "ok"} + if err != nil { + hh.Status = "failed" + hh.Error = err.Error() + anyError = true + } + resp = append(resp, hh) + } + if anyError { + w.WriteHeader(http.StatusServiceUnavailable) + } else { + w.WriteHeader(http.StatusOK) + } + RenderJSON(w, resp) + } + return http.HandlerFunc(fn) + } +} + +// Recoverer is a middleware that recovers from panics, logs the panic and returns a HTTP 500 status if possible. +func Recoverer(l logger.Backend) func(http.Handler) http.Handler { + return func(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + defer func() { + if rvr := recover(); rvr != nil { + l.Logf("request panic for %s from %s, %v", r.URL.String(), r.RemoteAddr, rvr) + if rvr != http.ErrAbortHandler { + l.Logf(string(debug.Stack())) + } + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + } + }() + h.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) + } +} + +// Headers middleware adds headers to request +func Headers(headers ...string) func(http.Handler) http.Handler { + + return func(h http.Handler) http.Handler { + + fn := func(w http.ResponseWriter, r *http.Request) { + for _, h := range headers { + elems := strings.Split(h, ":") + if len(elems) != 2 { + continue + } + r.Header.Set(strings.TrimSpace(elems[0]), strings.TrimSpace(elems[1])) + } + h.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) + } +} + +// Maybe middleware will allow you to change the flow of the middleware stack execution depending on return +// value of maybeFn(request). This is useful for example if you'd like to skip a middleware handler if +// a request does not satisfy the maybeFn logic. +// borrowed from https://github.com/go-chi/chi/blob/master/middleware/maybe.go +func Maybe(mw func(http.Handler) http.Handler, maybeFn func(r *http.Request) bool) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if maybeFn(r) { + mw(next).ServeHTTP(w, r) + } else { + next.ServeHTTP(w, r) + } + }) + } +} + +// RealIP is a middleware that sets a http.Request's RemoteAddr to the results +// of parsing either the X-Forwarded-For or X-Real-IP headers. +// +// This middleware should only be used if user can trust the headers sent with request. +// If reverse proxies are configured to pass along arbitrary header values from the client, +// or if this middleware used without a reverse proxy, malicious clients could set anything +// as X-Forwarded-For header and attack the server in various ways. +func RealIP(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + if rip, err := realip.Get(r); err == nil { + r.RemoteAddr = rip + } + h.ServeHTTP(w, r) + } + + return http.HandlerFunc(fn) +} + +// Reject is a middleware that conditionally rejects requests with a given status code and message. +// user-defined condition function rejectFn is used to determine if the request should be rejected. +func Reject(errCode int, errMsg string, rejectFn func(r *http.Request) bool) func(h http.Handler) http.Handler { + return func(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + if rejectFn(r) { + http.Error(w, errMsg, errCode) + return + } + h.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) + } +} diff --git a/vendor/github.com/go-pkgz/rest/nocache.go b/vendor/github.com/go-pkgz/rest/nocache.go new file mode 100644 index 00000000..aa0fa946 --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/nocache.go @@ -0,0 +1,55 @@ +package rest + +import ( + "net/http" + "time" +) + +// borrowed from https://github.com/go-chi/chi/blob/master/middleware/nocache.go + +var epoch = time.Unix(0, 0).UTC().Format(time.RFC1123) + +var noCacheHeaders = map[string]string{ + "Expires": epoch, + "Cache-Control": "no-cache, no-store, no-transform, must-revalidate, private, max-age=0", + "Pragma": "no-cache", + "X-Accel-Expires": "0", +} + +var etagHeaders = []string{ + "ETag", + "If-Modified-Since", + "If-Match", + "If-None-Match", + "If-Range", + "If-Unmodified-Since", +} + +// NoCache is a simple piece of middleware that sets a number of HTTP headers to prevent +// a router (or subrouter) from being cached by an upstream proxy and/or client. +// +// As per http://wiki.nginx.org/HttpProxyModule - NoCache sets: +// Expires: Thu, 01 Jan 1970 00:00:00 UTC +// Cache-Control: no-cache, private, max-age=0 +// X-Accel-Expires: 0 +// Pragma: no-cache (for HTTP/1.0 proxies/clients) +func NoCache(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + + // Delete any ETag headers that may have been set + for _, v := range etagHeaders { + if r.Header.Get(v) != "" { + r.Header.Del(v) + } + } + + // Set our NoCache headers + for k, v := range noCacheHeaders { + w.Header().Set(k, v) + } + + h.ServeHTTP(w, r) + } + + return http.HandlerFunc(fn) +} diff --git a/vendor/github.com/go-pkgz/rest/onlyfrom.go b/vendor/github.com/go-pkgz/rest/onlyfrom.go new file mode 100644 index 00000000..cccd77ed --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/onlyfrom.go @@ -0,0 +1,59 @@ +package rest + +import ( + "fmt" + "net" + "net/http" + "strings" + + "github.com/go-pkgz/rest/realip" +) + +// OnlyFrom middleware allows access for limited list of source IPs. +// Such IPs can be defined as complete ip (like 192.168.1.12), prefix (129.168.) or CIDR (192.168.0.0/16) +func OnlyFrom(onlyIps ...string) func(http.Handler) http.Handler { + return func(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + if len(onlyIps) == 0 { + // no restrictions if no ips defined + h.ServeHTTP(w, r) + return + } + matched, ip, err := matchSourceIP(r, onlyIps) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + RenderJSON(w, JSON{"error": fmt.Sprintf("can't get realip: %s", err)}) + return + } + if matched { + // matched ip - allow + h.ServeHTTP(w, r) + return + } + + w.WriteHeader(http.StatusForbidden) + RenderJSON(w, JSON{"error": fmt.Sprintf("ip %q rejected", ip)}) + } + return http.HandlerFunc(fn) + } +} + +// matchSourceIP returns true if request's ip matches any of ips +func matchSourceIP(r *http.Request, ips []string) (result bool, match string, err error) { + ip, err := realip.Get(r) + if err != nil { + return false, "", fmt.Errorf("can't get realip: %w", err) // we can't get ip, so no match + } + // check for ip prefix or CIDR + for _, exclIP := range ips { + if _, cidrnet, err := net.ParseCIDR(exclIP); err == nil { + if cidrnet.Contains(net.ParseIP(ip)) { + return true, ip, nil + } + } + if strings.HasPrefix(ip, exclIP) { + return true, ip, nil + } + } + return false, ip, nil +} diff --git a/vendor/github.com/go-pkgz/rest/profiler.go b/vendor/github.com/go-pkgz/rest/profiler.go new file mode 100644 index 00000000..88d8f12a --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/profiler.go @@ -0,0 +1,48 @@ +package rest + +import ( + "expvar" + "fmt" + "net/http" + "net/http/pprof" +) + +// Profiler is a convenient subrouter used for mounting net/http/pprof. ie. +// +// func MyService() http.Handler { +// r := chi.NewRouter() +// // ..middlewares +// r.Mount("/debug", middleware.Profiler()) +// // ..routes +// return r +// } +func Profiler(onlyIps ...string) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/pprof/", pprof.Index) + mux.HandleFunc("/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/pprof/profile", pprof.Profile) + mux.HandleFunc("/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/pprof/trace", pprof.Trace) + mux.Handle("/pprof/block", pprof.Handler("block")) + mux.Handle("/pprof/heap", pprof.Handler("heap")) + mux.Handle("/pprof/goroutine", pprof.Handler("goroutine")) + mux.Handle("/pprof/threadcreate", pprof.Handler("threadcreate")) + mux.HandleFunc("/vars", expVars) + + return Wrap(mux, NoCache, OnlyFrom(onlyIps...)) +} + +// expVars copied from stdlib expvar.go as is not public. +func expVars(w http.ResponseWriter, _ *http.Request) { + first := true + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, "{\n") + expvar.Do(func(kv expvar.KeyValue) { + if !first { + fmt.Fprintf(w, ",\n") + } + first = false + fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value) + }) + fmt.Fprintf(w, "\n}\n") +} diff --git a/vendor/github.com/go-pkgz/rest/realip/real.go b/vendor/github.com/go-pkgz/rest/realip/real.go new file mode 100644 index 00000000..c830a10f --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/realip/real.go @@ -0,0 +1,81 @@ +// Package realip extracts a real IP address from the request. +package realip + +import ( + "bytes" + "fmt" + "net" + "net/http" + "strings" +) + +type ipRange struct { + start net.IP + end net.IP +} + +var privateRanges = []ipRange{ + {start: net.ParseIP("10.0.0.0"), end: net.ParseIP("10.255.255.255")}, + {start: net.ParseIP("100.64.0.0"), end: net.ParseIP("100.127.255.255")}, + {start: net.ParseIP("172.16.0.0"), end: net.ParseIP("172.31.255.255")}, + {start: net.ParseIP("192.0.0.0"), end: net.ParseIP("192.0.0.255")}, + {start: net.ParseIP("192.168.0.0"), end: net.ParseIP("192.168.255.255")}, + {start: net.ParseIP("198.18.0.0"), end: net.ParseIP("198.19.255.255")}, + {start: net.ParseIP("::1"), end: net.ParseIP("::1")}, + {start: net.ParseIP("fc00::"), end: net.ParseIP("fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff")}, + {start: net.ParseIP("fe80::"), end: net.ParseIP("febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff")}, +} + +// Get returns real ip from the given request +// Prioritize public IPs over private IPs +func Get(r *http.Request) (string, error) { + var firstIP string + for _, h := range []string{"X-Forwarded-For", "X-Real-Ip"} { + addresses := strings.Split(r.Header.Get(h), ",") + for i := len(addresses) - 1; i >= 0; i-- { + ip := strings.TrimSpace(addresses[i]) + realIP := net.ParseIP(ip) + if firstIP == "" && realIP != nil { + firstIP = ip + } + if !realIP.IsGlobalUnicast() || isPrivateSubnet(realIP) { + continue + } + return ip, nil + } + } + + if firstIP != "" { + return firstIP, nil + } + + ip, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return "", fmt.Errorf("can't parse ip %q: %w", r.RemoteAddr, err) + } + if netIP := net.ParseIP(ip); netIP == nil { + return "", fmt.Errorf("no valid ip found") + } + + return ip, nil +} + +// isPrivateSubnet - check to see if this ip is in a private subnet +func isPrivateSubnet(ipAddress net.IP) bool { + + // inRange - check to see if a given ip address is within a range given + inRange := func(r ipRange, ipAddress net.IP) bool { + // ensure the IPs are in the same format for comparison + ipAddress = ipAddress.To16() + r.start = r.start.To16() + r.end = r.end.To16() + return bytes.Compare(ipAddress, r.start) >= 0 && bytes.Compare(ipAddress, r.end) <= 0 + } + + for _, r := range privateRanges { + if inRange(r, ipAddress) { + return true + } + } + return false +} diff --git a/vendor/github.com/go-pkgz/rest/rest.go b/vendor/github.com/go-pkgz/rest/rest.go new file mode 100644 index 00000000..1b321ec0 --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/rest.go @@ -0,0 +1,99 @@ +// Package rest provides common middlewares and helpers for rest services +package rest + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "time" +) + +// JSON is a map alias, just for convenience +type JSON map[string]interface{} + +// RenderJSON sends data as json +func RenderJSON(w http.ResponseWriter, data interface{}) { + buf := &bytes.Buffer{} + enc := json.NewEncoder(buf) + enc.SetEscapeHTML(true) + if err := enc.Encode(data); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json; charset=utf-8") + _, _ = w.Write(buf.Bytes()) +} + +// RenderJSONFromBytes sends binary data as json +func RenderJSONFromBytes(w http.ResponseWriter, r *http.Request, data []byte) error { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + if _, err := w.Write(data); err != nil { + return fmt.Errorf("failed to send response to %s: %w", r.RemoteAddr, err) + } + return nil +} + +// RenderJSONWithHTML allows html tags and forces charset=utf-8 +func RenderJSONWithHTML(w http.ResponseWriter, r *http.Request, v interface{}) error { + + encodeJSONWithHTML := func(v interface{}) ([]byte, error) { + buf := &bytes.Buffer{} + enc := json.NewEncoder(buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(v); err != nil { + return nil, fmt.Errorf("json encoding failed: %w", err) + } + return buf.Bytes(), nil + } + + data, err := encodeJSONWithHTML(v) + if err != nil { + return err + } + return RenderJSONFromBytes(w, r, data) +} + +// renderJSONWithStatus sends data as json and enforces status code +func renderJSONWithStatus(w http.ResponseWriter, data interface{}, code int) { + buf := &bytes.Buffer{} + enc := json.NewEncoder(buf) + enc.SetEscapeHTML(true) + if err := enc.Encode(data); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(code) + _, _ = w.Write(buf.Bytes()) +} + +// ParseFromTo parses from and to query params of the request +func ParseFromTo(r *http.Request) (from, to time.Time, err error) { + parseTimeStamp := func(ts string) (time.Time, error) { + formats := []string{ + "2006-01-02T15:04:05.000000000", + "2006-01-02T15:04:05", + "2006-01-02T15:04", + "20060102", + time.RFC3339, + time.RFC3339Nano, + } + + for _, f := range formats { + if t, e := time.Parse(f, ts); e == nil { + return t, nil + } + } + return time.Time{}, fmt.Errorf("can't parse date %q", ts) + } + + if from, err = parseTimeStamp(r.URL.Query().Get("from")); err != nil { + return from, to, fmt.Errorf("incorrect from time: %w", err) + } + + if to, err = parseTimeStamp(r.URL.Query().Get("to")); err != nil { + return from, to, fmt.Errorf("incorrect to time: %w", err) + } + return from, to, nil +} diff --git a/vendor/github.com/go-pkgz/rest/rewrite.go b/vendor/github.com/go-pkgz/rest/rewrite.go new file mode 100644 index 00000000..f3286bd2 --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/rewrite.go @@ -0,0 +1,57 @@ +package rest + +import ( + "context" + "net/http" + "net/url" + "path" + "regexp" + "strings" +) + +// Rewrite middleware with from->to rule. Supports regex (like nginx) and prevents multiple rewrites +// example: Rewrite(`^/sites/(.*)/settings/$`, `/sites/settings/$1` +func Rewrite(from, to string) func(http.Handler) http.Handler { + reFrom := regexp.MustCompile(from) + + f := func(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + + ctx := r.Context() + + // prevent double rewrites + if ctx != nil { + if _, ok := ctx.Value(contextKey("rewrite")).(bool); ok { + next.ServeHTTP(w, r) + return + } + } + + if !reFrom.MatchString(r.URL.Path) { + next.ServeHTTP(w, r) + return + } + + ru := reFrom.ReplaceAllString(r.URL.Path, to) + cru := path.Clean(ru) + if strings.HasSuffix(ru, "/") { // don't drop trailing slash + cru += "/" + } + u, e := url.Parse(cru) + if e != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + r.Header.Set("X-Original-URL", r.URL.RequestURI()) + r.URL.Path = u.Path + r.URL.RawPath = u.RawPath + if u.RawQuery != "" { + r.URL.RawQuery = u.RawQuery + } + ctx = context.WithValue(ctx, contextKey("rewrite"), true) + next.ServeHTTP(w, r.WithContext(ctx)) + } + return http.HandlerFunc(fn) + } + return f +} diff --git a/vendor/github.com/go-pkgz/rest/sizelimit.go b/vendor/github.com/go-pkgz/rest/sizelimit.go new file mode 100644 index 00000000..cfa03188 --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/sizelimit.go @@ -0,0 +1,40 @@ +package rest + +import ( + "bytes" + "io" + "net/http" +) + +// SizeLimit middleware checks if body size is above the limit and returns StatusRequestEntityTooLarge (413) +func SizeLimit(size int64) func(http.Handler) http.Handler { + + return func(h http.Handler) http.Handler { + + fn := func(w http.ResponseWriter, r *http.Request) { + + // check ContentLength + if r.ContentLength > size { + w.WriteHeader(http.StatusRequestEntityTooLarge) + return + } + + // check size of the actual body + content, err := io.ReadAll(io.LimitReader(r.Body, size+1)) + if err != nil { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + _ = r.Body.Close() // the original body already consumed + + if int64(len(content)) > size { + w.WriteHeader(http.StatusRequestEntityTooLarge) + return + } + r.Body = io.NopCloser(bytes.NewReader(content)) + h.ServeHTTP(w, r) + } + + return http.HandlerFunc(fn) + } +} diff --git a/vendor/github.com/go-pkgz/rest/throttle.go b/vendor/github.com/go-pkgz/rest/throttle.go new file mode 100644 index 00000000..9a5c42ae --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/throttle.go @@ -0,0 +1,46 @@ +package rest + +import ( + "net/http" +) + +// Throttle middleware checks how many request in-fly and rejects with 503 if exceeded +func Throttle(limit int64) func(http.Handler) http.Handler { + + ch := make(chan struct{}, limit) + return func(h http.Handler) http.Handler { + + fn := func(w http.ResponseWriter, r *http.Request) { + + if limit <= 0 { + h.ServeHTTP(w, r) + return + } + + var acquired bool + defer func() { + if !acquired { + return + } + select { + case <-ch: + return + default: + return + } + }() + + select { + case ch <- struct{}{}: + acquired = true + h.ServeHTTP(w, r) + return + default: + w.WriteHeader(http.StatusServiceUnavailable) + return + } + } + + return http.HandlerFunc(fn) + } +} diff --git a/vendor/github.com/go-pkgz/rest/trace.go b/vendor/github.com/go-pkgz/rest/trace.go new file mode 100644 index 00000000..77d14474 --- /dev/null +++ b/vendor/github.com/go-pkgz/rest/trace.go @@ -0,0 +1,48 @@ +package rest + +import ( + "context" + "crypto/rand" + "crypto/sha1" //nolint:gosec //not used for cryptography + "encoding/hex" + "fmt" + "net/http" + "time" +) + +type contextKey string + +const traceHeader = "X-Request-ID" + +// Trace looks for header X-Request-ID and makes it as random id if not found, then populates it to the result's header +// and to request context +func Trace(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + traceID := r.Header.Get(traceHeader) + if traceID == "" { + traceID = randToken() + } + w.Header().Set(traceHeader, traceID) + ctx := context.WithValue(r.Context(), contextKey("requestID"), traceID) + r = r.WithContext(ctx) + next.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) +} + +// GetTraceID returns request id from the context +func GetTraceID(r *http.Request) string { + if id, ok := r.Context().Value(contextKey("requestID")).(string); ok { + return id + } + return "" +} + +func randToken() string { + b := make([]byte, 32) + if _, err := rand.Read(b); err != nil { + return fmt.Sprintf("%x", time.Now().Nanosecond()) + } + sum := sha1.Sum(b) //nolint:gosec //not used for cryptography + return hex.EncodeToString(sum[:]) +} diff --git a/vendor/github.com/gorilla/css/LICENSE b/vendor/github.com/gorilla/css/LICENSE deleted file mode 100644 index ee0d53ce..00000000 --- a/vendor/github.com/gorilla/css/LICENSE +++ /dev/null @@ -1,28 +0,0 @@ -Copyright (c) 2023 The Gorilla Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/vendor/github.com/gorilla/css/scanner/doc.go b/vendor/github.com/gorilla/css/scanner/doc.go deleted file mode 100644 index f19850e1..00000000 --- a/vendor/github.com/gorilla/css/scanner/doc.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2012 The Gorilla Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -/* -Package gorilla/css/scanner generates tokens for a CSS3 input. - -It follows the CSS3 specification located at: - - http://www.w3.org/TR/css3-syntax/ - -To use it, create a new scanner for a given CSS string and call Next() until -the token returned has type TokenEOF or TokenError: - - s := scanner.New(myCSS) - for { - token := s.Next() - if token.Type == scanner.TokenEOF || token.Type == scanner.TokenError { - break - } - // Do something with the token... - } - -Following the CSS3 specification, an error can only occur when the scanner -finds an unclosed quote or unclosed comment. In these cases the text becomes -"untokenizable". Everything else is tokenizable and it is up to a parser -to make sense of the token stream (or ignore nonsensical token sequences). - -Note: the scanner doesn't perform lexical analysis or, in other words, it -doesn't care about the token context. It is intended to be used by a -lexer or parser. -*/ -package scanner diff --git a/vendor/github.com/gorilla/css/scanner/scanner.go b/vendor/github.com/gorilla/css/scanner/scanner.go deleted file mode 100644 index 25a7c657..00000000 --- a/vendor/github.com/gorilla/css/scanner/scanner.go +++ /dev/null @@ -1,360 +0,0 @@ -// Copyright 2012 The Gorilla Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package scanner - -import ( - "fmt" - "regexp" - "strings" - "unicode" - "unicode/utf8" -) - -// tokenType identifies the type of lexical tokens. -type tokenType int - -// String returns a string representation of the token type. -func (t tokenType) String() string { - return tokenNames[t] -} - -// Token represents a token and the corresponding string. -type Token struct { - Type tokenType - Value string - Line int - Column int -} - -// String returns a string representation of the token. -func (t *Token) String() string { - if len(t.Value) > 10 { - return fmt.Sprintf("%s (line: %d, column: %d): %.10q...", - t.Type, t.Line, t.Column, t.Value) - } - return fmt.Sprintf("%s (line: %d, column: %d): %q", - t.Type, t.Line, t.Column, t.Value) -} - -// All tokens ----------------------------------------------------------------- - -// The complete list of tokens in CSS3. -const ( - // Scanner flags. - TokenError tokenType = iota - TokenEOF - // From now on, only tokens from the CSS specification. - TokenIdent - TokenAtKeyword - TokenString - TokenHash - TokenNumber - TokenPercentage - TokenDimension - TokenURI - TokenUnicodeRange - TokenCDO - TokenCDC - TokenS - TokenComment - TokenFunction - TokenIncludes - TokenDashMatch - TokenPrefixMatch - TokenSuffixMatch - TokenSubstringMatch - TokenChar - TokenBOM -) - -// tokenNames maps tokenType's to their names. Used for conversion to string. -var tokenNames = map[tokenType]string{ - TokenError: "error", - TokenEOF: "EOF", - TokenIdent: "IDENT", - TokenAtKeyword: "ATKEYWORD", - TokenString: "STRING", - TokenHash: "HASH", - TokenNumber: "NUMBER", - TokenPercentage: "PERCENTAGE", - TokenDimension: "DIMENSION", - TokenURI: "URI", - TokenUnicodeRange: "UNICODE-RANGE", - TokenCDO: "CDO", - TokenCDC: "CDC", - TokenS: "S", - TokenComment: "COMMENT", - TokenFunction: "FUNCTION", - TokenIncludes: "INCLUDES", - TokenDashMatch: "DASHMATCH", - TokenPrefixMatch: "PREFIXMATCH", - TokenSuffixMatch: "SUFFIXMATCH", - TokenSubstringMatch: "SUBSTRINGMATCH", - TokenChar: "CHAR", - TokenBOM: "BOM", -} - -// Macros and productions ----------------------------------------------------- -// http://www.w3.org/TR/css3-syntax/#tokenization - -var macroRegexp = regexp.MustCompile(`\{[a-z]+\}`) - -// macros maps macro names to patterns to be expanded. -var macros = map[string]string{ - // must be escaped: `\.+*?()|[]{}^$` - "ident": `-?{nmstart}{nmchar}*`, - "name": `{nmchar}+`, - "nmstart": `[a-zA-Z_]|{nonascii}|{escape}`, - "nonascii": "[\u0080-\uD7FF\uE000-\uFFFD\U00010000-\U0010FFFF]", - "unicode": `\\[0-9a-fA-F]{1,6}{wc}?`, - "escape": "{unicode}|\\\\[\u0020-\u007E\u0080-\uD7FF\uE000-\uFFFD\U00010000-\U0010FFFF]", - "nmchar": `[a-zA-Z0-9_-]|{nonascii}|{escape}`, - "num": `[0-9]*\.[0-9]+|[0-9]+`, - "string": `"(?:{stringchar}|')*"|'(?:{stringchar}|")*'`, - "stringchar": `{urlchar}|[ ]|\\{nl}`, - "nl": `[\n\r\f]|\r\n`, - "w": `{wc}*`, - "wc": `[\t\n\f\r ]`, - - // urlchar should accept [(ascii characters minus those that need escaping)|{nonascii}|{escape}] - // ASCII characters range = `[\u0020-\u007e]` - // Skip space \u0020 = `[\u0021-\u007e]` - // Skip quotation mark \0022 = `[\u0021\u0023-\u007e]` - // Skip apostrophe \u0027 = `[\u0021\u0023-\u0026\u0028-\u007e]` - // Skip reverse solidus \u005c = `[\u0021\u0023-\u0026\u0028-\u005b\u005d\u007e]` - // Finally, the left square bracket (\u005b) and right (\u005d) needs escaping themselves - "urlchar": "[\u0021\u0023-\u0026\u0028-\\\u005b\\\u005d-\u007E]|{nonascii}|{escape}", -} - -// productions maps the list of tokens to patterns to be expanded. -var productions = map[tokenType]string{ - // Unused regexps (matched using other methods) are commented out. - TokenIdent: `{ident}`, - TokenAtKeyword: `@{ident}`, - TokenString: `{string}`, - TokenHash: `#{name}`, - TokenNumber: `{num}`, - TokenPercentage: `{num}%`, - TokenDimension: `{num}{ident}`, - TokenURI: `url\({w}(?:{string}|{urlchar}*?){w}\)`, - TokenUnicodeRange: `U\+[0-9A-F\?]{1,6}(?:-[0-9A-F]{1,6})?`, - //TokenCDO: ``, - TokenS: `{wc}+`, - TokenComment: `/\*[^\*]*[\*]+(?:[^/][^\*]*[\*]+)*/`, - TokenFunction: `{ident}\(`, - //TokenIncludes: `~=`, - //TokenDashMatch: `\|=`, - //TokenPrefixMatch: `\^=`, - //TokenSuffixMatch: `\$=`, - //TokenSubstringMatch: `\*=`, - //TokenChar: `[^"']`, - //TokenBOM: "\uFEFF", -} - -// matchers maps the list of tokens to compiled regular expressions. -// -// The map is filled on init() using the macros and productions defined in -// the CSS specification. -var matchers = map[tokenType]*regexp.Regexp{} - -// matchOrder is the order to test regexps when first-char shortcuts -// can't be used. -var matchOrder = []tokenType{ - TokenURI, - TokenFunction, - TokenUnicodeRange, - TokenIdent, - TokenDimension, - TokenPercentage, - TokenNumber, - TokenCDC, -} - -func init() { - // replace macros and compile regexps for productions. - replaceMacro := func(s string) string { - return "(?:" + macros[s[1:len(s)-1]] + ")" - } - for t, s := range productions { - for macroRegexp.MatchString(s) { - s = macroRegexp.ReplaceAllStringFunc(s, replaceMacro) - } - matchers[t] = regexp.MustCompile("^(?:" + s + ")") - } -} - -// Scanner -------------------------------------------------------------------- - -// New returns a new CSS scanner for the given input. -func New(input string) *Scanner { - // Normalize newlines. - // https://www.w3.org/TR/css-syntax-3/#input-preprocessing - input = strings.Replace(input, "\r\n", "\n", -1) - input = strings.Replace(input, "\r", "\n", -1) - input = strings.Replace(input, "\f", "\n", -1) - input = strings.Replace(input, "\u0000", "\ufffd", -1) - return &Scanner{ - input: input, - row: 1, - col: 1, - } -} - -// Scanner scans an input and emits tokens following the CSS3 specification. -type Scanner struct { - input string - pos int - row int - col int - err *Token -} - -// Next returns the next token from the input. -// -// At the end of the input the token type is TokenEOF. -// -// If the input can't be tokenized the token type is TokenError. This occurs -// in case of unclosed quotation marks or comments. -func (s *Scanner) Next() *Token { - if s.err != nil { - return s.err - } - if s.pos >= len(s.input) { - s.err = &Token{TokenEOF, "", s.row, s.col} - return s.err - } - if s.pos == 0 { - // Test BOM only once, at the beginning of the file. - if strings.HasPrefix(s.input, "\uFEFF") { - return s.emitSimple(TokenBOM, "\uFEFF") - } - } - // There's a lot we can guess based on the first byte so we'll take a - // shortcut before testing multiple regexps. - input := s.input[s.pos:] - switch input[0] { - case '\t', '\n', ' ': - // Whitespace. - return s.emitToken(TokenS, matchers[TokenS].FindString(input)) - case '.': - // Dot is too common to not have a quick check. - // We'll test if this is a Char; if it is followed by a number it is a - // dimension/percentage/number, and this will be matched later. - if len(input) > 1 && !unicode.IsDigit(rune(input[1])) { - return s.emitSimple(TokenChar, ".") - } - case '#': - // Another common one: Hash or Char. - if match := matchers[TokenHash].FindString(input); match != "" { - return s.emitToken(TokenHash, match) - } - return s.emitSimple(TokenChar, "#") - case '@': - // Another common one: AtKeyword or Char. - if match := matchers[TokenAtKeyword].FindString(input); match != "" { - return s.emitSimple(TokenAtKeyword, match) - } - return s.emitSimple(TokenChar, "@") - case ':', ',', ';', '%', '&', '+', '=', '>', '(', ')', '[', ']', '{', '}': - // More common chars. - return s.emitSimple(TokenChar, string(input[0])) - case '"', '\'': - // String or error. - match := matchers[TokenString].FindString(input) - if match != "" { - return s.emitToken(TokenString, match) - } - - s.err = &Token{TokenError, "unclosed quotation mark", s.row, s.col} - return s.err - case '/': - // Comment, error or Char. - if len(input) > 1 && input[1] == '*' { - match := matchers[TokenComment].FindString(input) - if match != "" { - return s.emitToken(TokenComment, match) - } else { - s.err = &Token{TokenError, "unclosed comment", s.row, s.col} - return s.err - } - } - return s.emitSimple(TokenChar, "/") - case '~': - // Includes or Char. - return s.emitPrefixOrChar(TokenIncludes, "~=") - case '|': - // DashMatch or Char. - return s.emitPrefixOrChar(TokenDashMatch, "|=") - case '^': - // PrefixMatch or Char. - return s.emitPrefixOrChar(TokenPrefixMatch, "^=") - case '$': - // SuffixMatch or Char. - return s.emitPrefixOrChar(TokenSuffixMatch, "$=") - case '*': - // SubstringMatch or Char. - return s.emitPrefixOrChar(TokenSubstringMatch, "*=") - case '<': - // CDO or Char. - return s.emitPrefixOrChar(TokenCDO, " which includes the use of that to permit -// conditionals as per https://docs.microsoft.com/en-us/previous-versions/windows/internet-explorer/ie-developer/compatibility/ms537512(v=vs.85)?redirectedfrom=MSDN -// -// What is not permitted are CDATA XML comments, as the x/net/html package we depend -// on does not handle this fully and we are not choosing to take on that work: -// https://pkg.go.dev/golang.org/x/net/html#Tokenizer.AllowCDATA . If the x/net/html -// package changes this then these will be considered, otherwise if you AllowComments -// but provide a CDATA comment, then as per the documentation in x/net/html this will -// be treated as a plain HTML comment. -func (p *Policy) AllowComments() { - p.allowComments = true -} - -// AllowNoAttrs says that attributes on element are optional. -// -// The attribute policy is only added to the core policy when OnElements(...) -// are called. -func (p *Policy) AllowNoAttrs() *attrPolicyBuilder { - - p.init() - - abp := attrPolicyBuilder{ - p: p, - allowEmpty: true, - } - return &abp -} - -// AllowNoAttrs says that attributes on element are optional. -// -// The attribute policy is only added to the core policy when OnElements(...) -// are called. -func (abp *attrPolicyBuilder) AllowNoAttrs() *attrPolicyBuilder { - - abp.allowEmpty = true - - return abp -} - -// Matching allows a regular expression to be applied to a nascent attribute -// policy, and returns the attribute policy. -func (abp *attrPolicyBuilder) Matching(regex *regexp.Regexp) *attrPolicyBuilder { - - abp.regexp = regex - - return abp -} - -// OnElements will bind an attribute policy to a given range of HTML elements -// and return the updated policy -func (abp *attrPolicyBuilder) OnElements(elements ...string) *Policy { - - for _, element := range elements { - element = strings.ToLower(element) - - for _, attr := range abp.attrNames { - - if _, ok := abp.p.elsAndAttrs[element]; !ok { - abp.p.elsAndAttrs[element] = make(map[string][]attrPolicy) - } - - ap := attrPolicy{} - if abp.regexp != nil { - ap.regexp = abp.regexp - } - - abp.p.elsAndAttrs[element][attr] = append(abp.p.elsAndAttrs[element][attr], ap) - } - - if abp.allowEmpty { - abp.p.setOfElementsAllowedWithoutAttrs[element] = struct{}{} - - if _, ok := abp.p.elsAndAttrs[element]; !ok { - abp.p.elsAndAttrs[element] = make(map[string][]attrPolicy) - } - } - } - - return abp.p -} - -// OnElementsMatching will bind an attribute policy to all elements matching a given regex -// and return the updated policy -func (abp *attrPolicyBuilder) OnElementsMatching(regex *regexp.Regexp) *Policy { - for _, attr := range abp.attrNames { - if _, ok := abp.p.elsMatchingAndAttrs[regex]; !ok { - abp.p.elsMatchingAndAttrs[regex] = make(map[string][]attrPolicy) - } - ap := attrPolicy{} - if abp.regexp != nil { - ap.regexp = abp.regexp - } - abp.p.elsMatchingAndAttrs[regex][attr] = append(abp.p.elsMatchingAndAttrs[regex][attr], ap) - } - - if abp.allowEmpty { - abp.p.setOfElementsMatchingAllowedWithoutAttrs = append(abp.p.setOfElementsMatchingAllowedWithoutAttrs, regex) - if _, ok := abp.p.elsMatchingAndAttrs[regex]; !ok { - abp.p.elsMatchingAndAttrs[regex] = make(map[string][]attrPolicy) - } - } - - return abp.p -} - -// Globally will bind an attribute policy to all HTML elements and return the -// updated policy -func (abp *attrPolicyBuilder) Globally() *Policy { - - for _, attr := range abp.attrNames { - if _, ok := abp.p.globalAttrs[attr]; !ok { - abp.p.globalAttrs[attr] = []attrPolicy{} - } - - ap := attrPolicy{} - if abp.regexp != nil { - ap.regexp = abp.regexp - } - - abp.p.globalAttrs[attr] = append(abp.p.globalAttrs[attr], ap) - } - - return abp.p -} - -// AllowStyles takes a range of CSS property names and returns a -// style policy builder that allows you to specify the pattern and scope of -// the allowed property. -// -// The style policy is only added to the core policy when either Globally() -// or OnElements(...) are called. -func (p *Policy) AllowStyles(propertyNames ...string) *stylePolicyBuilder { - - p.init() - - abp := stylePolicyBuilder{ - p: p, - } - - for _, propertyName := range propertyNames { - abp.propertyNames = append(abp.propertyNames, strings.ToLower(propertyName)) - } - - return &abp -} - -// Matching allows a regular expression to be applied to a nascent style -// policy, and returns the style policy. -func (spb *stylePolicyBuilder) Matching(regex *regexp.Regexp) *stylePolicyBuilder { - - spb.regexp = regex - - return spb -} - -// MatchingEnum allows a list of allowed values to be applied to a nascent style -// policy, and returns the style policy. -func (spb *stylePolicyBuilder) MatchingEnum(enum ...string) *stylePolicyBuilder { - - spb.enum = enum - - return spb -} - -// MatchingHandler allows a handler to be applied to a nascent style -// policy, and returns the style policy. -func (spb *stylePolicyBuilder) MatchingHandler(handler func(string) bool) *stylePolicyBuilder { - - spb.handler = handler - - return spb -} - -// OnElements will bind a style policy to a given range of HTML elements -// and return the updated policy -func (spb *stylePolicyBuilder) OnElements(elements ...string) *Policy { - - for _, element := range elements { - element = strings.ToLower(element) - - for _, attr := range spb.propertyNames { - - if _, ok := spb.p.elsAndStyles[element]; !ok { - spb.p.elsAndStyles[element] = make(map[string][]stylePolicy) - } - - sp := stylePolicy{} - if spb.handler != nil { - sp.handler = spb.handler - } else if len(spb.enum) > 0 { - sp.enum = spb.enum - } else if spb.regexp != nil { - sp.regexp = spb.regexp - } else { - sp.handler = css.GetDefaultHandler(attr) - } - spb.p.elsAndStyles[element][attr] = append(spb.p.elsAndStyles[element][attr], sp) - } - } - - return spb.p -} - -// OnElementsMatching will bind a style policy to any HTML elements matching the pattern -// and return the updated policy -func (spb *stylePolicyBuilder) OnElementsMatching(regex *regexp.Regexp) *Policy { - - for _, attr := range spb.propertyNames { - - if _, ok := spb.p.elsMatchingAndStyles[regex]; !ok { - spb.p.elsMatchingAndStyles[regex] = make(map[string][]stylePolicy) - } - - sp := stylePolicy{} - if spb.handler != nil { - sp.handler = spb.handler - } else if len(spb.enum) > 0 { - sp.enum = spb.enum - } else if spb.regexp != nil { - sp.regexp = spb.regexp - } else { - sp.handler = css.GetDefaultHandler(attr) - } - spb.p.elsMatchingAndStyles[regex][attr] = append(spb.p.elsMatchingAndStyles[regex][attr], sp) - } - - return spb.p -} - -// Globally will bind a style policy to all HTML elements and return the -// updated policy -func (spb *stylePolicyBuilder) Globally() *Policy { - - for _, attr := range spb.propertyNames { - if _, ok := spb.p.globalStyles[attr]; !ok { - spb.p.globalStyles[attr] = []stylePolicy{} - } - - // Use only one strategy for validating styles, fallback to default - sp := stylePolicy{} - if spb.handler != nil { - sp.handler = spb.handler - } else if len(spb.enum) > 0 { - sp.enum = spb.enum - } else if spb.regexp != nil { - sp.regexp = spb.regexp - } else { - sp.handler = css.GetDefaultHandler(attr) - } - spb.p.globalStyles[attr] = append(spb.p.globalStyles[attr], sp) - } - - return spb.p -} - -// AllowElements will append HTML elements to the allowlist without applying an -// attribute policy to those elements (the elements are permitted -// sans-attributes) -func (p *Policy) AllowElements(names ...string) *Policy { - p.init() - - for _, element := range names { - element = strings.ToLower(element) - - if _, ok := p.elsAndAttrs[element]; !ok { - p.elsAndAttrs[element] = make(map[string][]attrPolicy) - } - } - - return p -} - -// AllowElementsMatching will append HTML elements to the allowlist if they -// match a regexp. -func (p *Policy) AllowElementsMatching(regex *regexp.Regexp) *Policy { - p.init() - if _, ok := p.elsMatchingAndAttrs[regex]; !ok { - p.elsMatchingAndAttrs[regex] = make(map[string][]attrPolicy) - } - return p -} - -// AllowURLSchemesMatching will append URL schemes to the allowlist if they -// match a regexp. -func (p *Policy) AllowURLSchemesMatching(r *regexp.Regexp) *Policy { - p.allowURLSchemeRegexps = append(p.allowURLSchemeRegexps, r) - return p -} - -// RewriteSrc will rewrite the src attribute of a resource downloading tag -// (e.g. , tag. -func (p *Policy) addDefaultSkipElementContent() { - p.init() - - p.setOfElementsToSkipContent["frame"] = struct{}{} - p.setOfElementsToSkipContent["frameset"] = struct{}{} - p.setOfElementsToSkipContent["iframe"] = struct{}{} - p.setOfElementsToSkipContent["noembed"] = struct{}{} - p.setOfElementsToSkipContent["noframes"] = struct{}{} - p.setOfElementsToSkipContent["noscript"] = struct{}{} - p.setOfElementsToSkipContent["nostyle"] = struct{}{} - p.setOfElementsToSkipContent["object"] = struct{}{} - p.setOfElementsToSkipContent["script"] = struct{}{} - p.setOfElementsToSkipContent["style"] = struct{}{} - p.setOfElementsToSkipContent["title"] = struct{}{} -} diff --git a/vendor/github.com/microcosm-cc/bluemonday/sanitize.go b/vendor/github.com/microcosm-cc/bluemonday/sanitize.go deleted file mode 100644 index 1f8d8552..00000000 --- a/vendor/github.com/microcosm-cc/bluemonday/sanitize.go +++ /dev/null @@ -1,1089 +0,0 @@ -// Copyright (c) 2014, David Kitchen -// -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// * Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. -// -// * Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// * Neither the name of the organisation (Microcosm) nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package bluemonday - -import ( - "bytes" - "fmt" - "io" - "net/url" - "regexp" - "strconv" - "strings" - - "golang.org/x/net/html" - - "github.com/aymerick/douceur/parser" -) - -var ( - dataAttribute = regexp.MustCompile("^data-.+") - dataAttributeXMLPrefix = regexp.MustCompile("^xml.+") - dataAttributeInvalidChars = regexp.MustCompile("[A-Z;]+") - cssUnicodeChar = regexp.MustCompile(`\\[0-9a-f]{1,6} ?`) - dataURIbase64Prefix = regexp.MustCompile(`^data:[^,]*;base64,`) -) - -// Sanitize takes a string that contains a HTML fragment or document and applies -// the given policy allowlist. -// -// It returns a HTML string that has been sanitized by the policy or an empty -// string if an error has occurred (most likely as a consequence of extremely -// malformed input) -func (p *Policy) Sanitize(s string) string { - if strings.TrimSpace(s) == "" { - return s - } - - return p.sanitizeWithBuff(strings.NewReader(s)).String() -} - -// SanitizeBytes takes a []byte that contains a HTML fragment or document and applies -// the given policy allowlist. -// -// It returns a []byte containing the HTML that has been sanitized by the policy -// or an empty []byte if an error has occurred (most likely as a consequence of -// extremely malformed input) -func (p *Policy) SanitizeBytes(b []byte) []byte { - if len(bytes.TrimSpace(b)) == 0 { - return b - } - - return p.sanitizeWithBuff(bytes.NewReader(b)).Bytes() -} - -// SanitizeReader takes an io.Reader that contains a HTML fragment or document -// and applies the given policy allowlist. -// -// It returns a bytes.Buffer containing the HTML that has been sanitized by the -// policy. Errors during sanitization will merely return an empty result. -func (p *Policy) SanitizeReader(r io.Reader) *bytes.Buffer { - return p.sanitizeWithBuff(r) -} - -// SanitizeReaderToWriter takes an io.Reader that contains a HTML fragment or document -// and applies the given policy allowlist and writes to the provided writer returning -// an error if there is one. -func (p *Policy) SanitizeReaderToWriter(r io.Reader, w io.Writer) error { - return p.sanitize(r, w) -} - -// Query represents a single part of the query string, a query param -type Query struct { - Key string - Value string - HasValue bool -} - -func parseQuery(query string) (values []Query, err error) { - // This is essentially a copy of parseQuery from - // https://golang.org/src/net/url/url.go but adjusted to build our values - // based on our type, which we need to preserve the ordering of the query - // string - for query != "" { - key := query - if i := strings.IndexAny(key, "&;"); i >= 0 { - key, query = key[:i], key[i+1:] - } else { - query = "" - } - if key == "" { - continue - } - value := "" - hasValue := false - if i := strings.Index(key, "="); i >= 0 { - key, value = key[:i], key[i+1:] - hasValue = true - } - key, err1 := url.QueryUnescape(key) - if err1 != nil { - if err == nil { - err = err1 - } - continue - } - value, err1 = url.QueryUnescape(value) - if err1 != nil { - if err == nil { - err = err1 - } - continue - } - values = append(values, Query{ - Key: key, - Value: value, - HasValue: hasValue, - }) - } - return values, err -} - -func encodeQueries(queries []Query) string { - var buff bytes.Buffer - for i, query := range queries { - buff.WriteString(url.QueryEscape(query.Key)) - if query.HasValue { - buff.WriteString("=") - buff.WriteString(url.QueryEscape(query.Value)) - } - if i < len(queries)-1 { - buff.WriteString("&") - } - } - return buff.String() -} - -func sanitizedURL(val string) (string, error) { - u, err := url.Parse(val) - if err != nil { - return "", err - } - - // we use parseQuery but not u.Query to keep the order not change because - // url.Values is a map which has a random order. - queryValues, err := parseQuery(u.RawQuery) - if err != nil { - return "", err - } - // sanitize the url query params - for i, query := range queryValues { - queryValues[i].Key = html.EscapeString(query.Key) - } - u.RawQuery = encodeQueries(queryValues) - // u.String() will also sanitize host/scheme/user/pass - return u.String(), nil -} - -// Performs the actual sanitization process. -func (p *Policy) sanitizeWithBuff(r io.Reader) *bytes.Buffer { - var buff bytes.Buffer - if err := p.sanitize(r, &buff); err != nil { - return &bytes.Buffer{} - } - return &buff -} - -type asStringWriter struct { - io.Writer -} - -func (a *asStringWriter) WriteString(s string) (int, error) { - return a.Write([]byte(s)) -} - -func (p *Policy) sanitize(r io.Reader, w io.Writer) error { - // It is possible that the developer has created the policy via: - // p := bluemonday.Policy{} - // rather than: - // p := bluemonday.NewPolicy() - // If this is the case, and if they haven't yet triggered an action that - // would initialize the maps, then we need to do that. - p.init() - - buff, ok := w.(stringWriterWriter) - if !ok { - buff = &asStringWriter{w} - } - - var ( - skipElementContent bool - skippingElementsCount int64 - skipClosingTag bool - closingTagToSkipStack []string - mostRecentlyStartedToken string - ) - - tokenizer := html.NewTokenizer(r) - for { - if tokenizer.Next() == html.ErrorToken { - err := tokenizer.Err() - if err == io.EOF { - // End of input means end of processing - return nil - } - - // Raw tokenizer error - return err - } - - token := tokenizer.Token() - switch token.Type { - case html.DoctypeToken: - - // DocType is not handled as there is no safe parsing mechanism - // provided by golang.org/x/net/html for the content, and this can - // be misused to insert HTML tags that are not then sanitized - // - // One might wish to recursively sanitize here using the same policy - // but I will need to do some further testing before considering - // this. - - case html.CommentToken: - - // Comments are ignored by default - if p.allowComments { - // But if allowed then write the comment out as-is - buff.WriteString(token.String()) - } - - case html.StartTagToken: - - mostRecentlyStartedToken = normaliseElementName(token.Data) - - switch normaliseElementName(token.Data) { - case `script`: - if !p.allowUnsafe { - continue - } - case `style`: - if !p.allowUnsafe { - continue - } - } - - aps, ok := p.elsAndAttrs[token.Data] - if !ok { - aa, matched := p.matchRegex(token.Data) - if !matched { - if _, ok := p.setOfElementsToSkipContent[token.Data]; ok { - skipElementContent = true - skippingElementsCount++ - } - if p.addSpaces { - if _, err := buff.WriteString(" "); err != nil { - return err - } - } - break - } - aps = aa - } - if len(token.Attr) != 0 { - token.Attr = p.sanitizeAttrs(token.Data, token.Attr, aps) - } - - if len(token.Attr) == 0 { - if !p.allowNoAttrs(token.Data) { - skipClosingTag = true - closingTagToSkipStack = append(closingTagToSkipStack, token.Data) - if p.addSpaces { - if _, err := buff.WriteString(" "); err != nil { - return err - } - } - break - } - } - - if !skipElementContent { - if _, err := buff.WriteString(token.String()); err != nil { - return err - } - } - - case html.EndTagToken: - - if mostRecentlyStartedToken == normaliseElementName(token.Data) { - mostRecentlyStartedToken = "" - } - - switch normaliseElementName(token.Data) { - case `script`: - if !p.allowUnsafe { - continue - } - case `style`: - if !p.allowUnsafe { - continue - } - } - - if skipClosingTag && closingTagToSkipStack[len(closingTagToSkipStack)-1] == token.Data { - closingTagToSkipStack = closingTagToSkipStack[:len(closingTagToSkipStack)-1] - if len(closingTagToSkipStack) == 0 { - skipClosingTag = false - } - if p.addSpaces { - if _, err := buff.WriteString(" "); err != nil { - return err - } - } - break - } - if _, ok := p.elsAndAttrs[token.Data]; !ok { - match := false - for regex := range p.elsMatchingAndAttrs { - if regex.MatchString(token.Data) { - skipElementContent = false - match = true - break - } - } - if _, ok := p.setOfElementsToSkipContent[token.Data]; ok && !match { - skippingElementsCount-- - if skippingElementsCount == 0 { - skipElementContent = false - } - } - if !match { - if p.addSpaces { - if _, err := buff.WriteString(" "); err != nil { - return err - } - } - break - } - } - - if !skipElementContent { - if _, err := buff.WriteString(token.String()); err != nil { - return err - } - } - - case html.SelfClosingTagToken: - - switch normaliseElementName(token.Data) { - case `script`: - if !p.allowUnsafe { - continue - } - case `style`: - if !p.allowUnsafe { - continue - } - } - - aps, ok := p.elsAndAttrs[token.Data] - if !ok { - aa, matched := p.matchRegex(token.Data) - if !matched { - if p.addSpaces && !matched { - if _, err := buff.WriteString(" "); err != nil { - return err - } - } - break - } - aps = aa - } - - if len(token.Attr) != 0 { - token.Attr = p.sanitizeAttrs(token.Data, token.Attr, aps) - } - - if len(token.Attr) == 0 && !p.allowNoAttrs(token.Data) { - if p.addSpaces { - if _, err := buff.WriteString(" "); err != nil { - return err - } - } - break - } - if !skipElementContent { - if _, err := buff.WriteString(token.String()); err != nil { - return err - } - } - - case html.TextToken: - - if !skipElementContent { - switch mostRecentlyStartedToken { - case `script`: - // not encouraged, but if a policy allows JavaScript we - // should not HTML escape it as that would break the output - // - // requires p.AllowUnsafe() - if p.allowUnsafe { - if _, err := buff.WriteString(token.Data); err != nil { - return err - } - } - case "style": - // not encouraged, but if a policy allows CSS styles we - // should not HTML escape it as that would break the output - // - // requires p.AllowUnsafe() - if p.allowUnsafe { - if _, err := buff.WriteString(token.Data); err != nil { - return err - } - } - default: - // HTML escape the text - if _, err := buff.WriteString(token.String()); err != nil { - return err - } - } - } - - default: - // A token that didn't exist in the html package when we wrote this - return fmt.Errorf("unknown token: %v", token) - } - } -} - -// sanitizeAttrs takes a set of element attribute policies and the global -// attribute policies and applies them to the []html.Attribute returning a set -// of html.Attributes that match the policies -func (p *Policy) sanitizeAttrs( - elementName string, - attrs []html.Attribute, - aps map[string][]attrPolicy, -) []html.Attribute { - - if len(attrs) == 0 { - return attrs - } - - hasStylePolicies := false - sps, elementHasStylePolicies := p.elsAndStyles[elementName] - if len(p.globalStyles) > 0 || (elementHasStylePolicies && len(sps) > 0) { - hasStylePolicies = true - } - // no specific element policy found, look for a pattern match - if !hasStylePolicies { - for k, v := range p.elsMatchingAndStyles { - if k.MatchString(elementName) { - if len(v) > 0 { - hasStylePolicies = true - break - } - } - } - } - - // Builds a new attribute slice based on the whether the attribute has been - // allowed explicitly or globally. - cleanAttrs := []html.Attribute{} -attrsLoop: - for _, htmlAttr := range attrs { - if p.allowDataAttributes { - // If we see a data attribute, let it through. - if isDataAttribute(htmlAttr.Key) { - cleanAttrs = append(cleanAttrs, htmlAttr) - continue - } - } - // Is this a "style" attribute, and if so, do we need to sanitize it? - if htmlAttr.Key == "style" && hasStylePolicies { - htmlAttr = p.sanitizeStyles(htmlAttr, elementName) - if htmlAttr.Val == "" { - // We've sanitized away any and all styles; don't bother to - // output the style attribute (even if it's allowed) - continue - } else { - cleanAttrs = append(cleanAttrs, htmlAttr) - continue - } - } - - // Is there an element specific attribute policy that applies? - if apl, ok := aps[htmlAttr.Key]; ok { - for _, ap := range apl { - if ap.regexp != nil { - if ap.regexp.MatchString(htmlAttr.Val) { - cleanAttrs = append(cleanAttrs, htmlAttr) - continue attrsLoop - } - } else { - cleanAttrs = append(cleanAttrs, htmlAttr) - continue attrsLoop - } - } - } - - // Is there a global attribute policy that applies? - if apl, ok := p.globalAttrs[htmlAttr.Key]; ok { - for _, ap := range apl { - if ap.regexp != nil { - if ap.regexp.MatchString(htmlAttr.Val) { - cleanAttrs = append(cleanAttrs, htmlAttr) - } - } else { - cleanAttrs = append(cleanAttrs, htmlAttr) - } - } - } - } - - if len(cleanAttrs) == 0 { - // If nothing was allowed, let's get out of here - return cleanAttrs - } - // cleanAttrs now contains the attributes that are permitted - - if linkable(elementName) { - if p.requireParseableURLs { - // Ensure URLs are parseable: - // - a.href - // - area.href - // - link.href - // - blockquote.cite - // - q.cite - // - img.src - // - script.src - tmpAttrs := []html.Attribute{} - for _, htmlAttr := range cleanAttrs { - switch elementName { - case "a", "area", "base", "link": - if htmlAttr.Key == "href" { - if u, ok := p.validURL(htmlAttr.Val); ok { - htmlAttr.Val = u - tmpAttrs = append(tmpAttrs, htmlAttr) - } - break - } - tmpAttrs = append(tmpAttrs, htmlAttr) - case "blockquote", "del", "ins", "q": - if htmlAttr.Key == "cite" { - if u, ok := p.validURL(htmlAttr.Val); ok { - htmlAttr.Val = u - tmpAttrs = append(tmpAttrs, htmlAttr) - } - break - } - tmpAttrs = append(tmpAttrs, htmlAttr) - case "audio", "embed", "iframe", "img", "script", "source", "track", "video": - if htmlAttr.Key == "src" { - if u, ok := p.validURL(htmlAttr.Val); ok { - if p.srcRewriter != nil { - parsedURL, err := url.Parse(u) - if err != nil { - fmt.Println(err) - } - p.srcRewriter(parsedURL) - u = parsedURL.String() - } - htmlAttr.Val = u - tmpAttrs = append(tmpAttrs, htmlAttr) - } - break - } - tmpAttrs = append(tmpAttrs, htmlAttr) - default: - tmpAttrs = append(tmpAttrs, htmlAttr) - } - } - cleanAttrs = tmpAttrs - } - - if (p.requireNoFollow || - p.requireNoFollowFullyQualifiedLinks || - p.requireNoReferrer || - p.requireNoReferrerFullyQualifiedLinks || - p.addTargetBlankToFullyQualifiedLinks) && - len(cleanAttrs) > 0 { - - // Add rel="nofollow" if a "href" exists - switch elementName { - case "a", "area", "base", "link": - var hrefFound bool - var externalLink bool - for _, htmlAttr := range cleanAttrs { - if htmlAttr.Key == "href" { - hrefFound = true - - u, err := url.Parse(htmlAttr.Val) - if err != nil { - continue - } - if u.Host != "" { - externalLink = true - } - - continue - } - } - - if hrefFound { - var ( - noFollowFound bool - noReferrerFound bool - targetBlankFound bool - ) - - addNoFollow := (p.requireNoFollow || - externalLink && p.requireNoFollowFullyQualifiedLinks) - - addNoReferrer := (p.requireNoReferrer || - externalLink && p.requireNoReferrerFullyQualifiedLinks) - - addTargetBlank := (externalLink && - p.addTargetBlankToFullyQualifiedLinks) - - tmpAttrs := []html.Attribute{} - for _, htmlAttr := range cleanAttrs { - - var appended bool - if htmlAttr.Key == "rel" && (addNoFollow || addNoReferrer) { - - if addNoFollow && !strings.Contains(htmlAttr.Val, "nofollow") { - htmlAttr.Val += " nofollow" - } - if addNoReferrer && !strings.Contains(htmlAttr.Val, "noreferrer") { - htmlAttr.Val += " noreferrer" - } - noFollowFound = addNoFollow - noReferrerFound = addNoReferrer - tmpAttrs = append(tmpAttrs, htmlAttr) - appended = true - } - - if elementName == "a" && htmlAttr.Key == "target" { - if htmlAttr.Val == "_blank" { - targetBlankFound = true - } - if addTargetBlank && !targetBlankFound { - htmlAttr.Val = "_blank" - targetBlankFound = true - tmpAttrs = append(tmpAttrs, htmlAttr) - appended = true - } - } - - if !appended { - tmpAttrs = append(tmpAttrs, htmlAttr) - } - } - if noFollowFound || noReferrerFound || targetBlankFound { - cleanAttrs = tmpAttrs - } - - if (addNoFollow && !noFollowFound) || (addNoReferrer && !noReferrerFound) { - rel := html.Attribute{} - rel.Key = "rel" - if addNoFollow { - rel.Val = "nofollow" - } - if addNoReferrer { - if rel.Val != "" { - rel.Val += " " - } - rel.Val += "noreferrer" - } - cleanAttrs = append(cleanAttrs, rel) - } - - if elementName == "a" && addTargetBlank && !targetBlankFound { - rel := html.Attribute{} - rel.Key = "target" - rel.Val = "_blank" - targetBlankFound = true - cleanAttrs = append(cleanAttrs, rel) - } - - if targetBlankFound { - // target="_blank" has a security risk that allows the - // opened window/tab to issue JavaScript calls against - // window.opener, which in effect allow the destination - // of the link to control the source: - // https://dev.to/ben/the-targetblank-vulnerability-by-example - // - // To mitigate this risk, we need to add a specific rel - // attribute if it is not already present. - // rel="noopener" - // - // Unfortunately this is processing the rel twice (we - // already looked at it earlier ^^) as we cannot be sure - // of the ordering of the href and rel, and whether we - // have fully satisfied that we need to do this. This - // double processing only happens *if* target="_blank" - // is true. - var noOpenerAdded bool - tmpAttrs := []html.Attribute{} - for _, htmlAttr := range cleanAttrs { - var appended bool - if htmlAttr.Key == "rel" { - if strings.Contains(htmlAttr.Val, "noopener") { - noOpenerAdded = true - tmpAttrs = append(tmpAttrs, htmlAttr) - } else { - htmlAttr.Val += " noopener" - noOpenerAdded = true - tmpAttrs = append(tmpAttrs, htmlAttr) - } - - appended = true - } - if !appended { - tmpAttrs = append(tmpAttrs, htmlAttr) - } - } - if noOpenerAdded { - cleanAttrs = tmpAttrs - } else { - // rel attr was not found, or else noopener would - // have been added already - rel := html.Attribute{} - rel.Key = "rel" - rel.Val = "noopener" - cleanAttrs = append(cleanAttrs, rel) - } - - } - } - default: - } - } - } - - if p.requireCrossOriginAnonymous && len(cleanAttrs) > 0 { - switch elementName { - case "audio", "img", "link", "script", "video": - var crossOriginFound bool - for _, htmlAttr := range cleanAttrs { - if htmlAttr.Key == "crossorigin" { - crossOriginFound = true - htmlAttr.Val = "anonymous" - } - } - - if !crossOriginFound { - crossOrigin := html.Attribute{} - crossOrigin.Key = "crossorigin" - crossOrigin.Val = "anonymous" - cleanAttrs = append(cleanAttrs, crossOrigin) - } - } - } - - if p.requireSandboxOnIFrame != nil && elementName == "iframe" { - var sandboxFound bool - for i, htmlAttr := range cleanAttrs { - if htmlAttr.Key == "sandbox" { - sandboxFound = true - var cleanVals []string - cleanValsSet := make(map[string]bool) - for _, val := range strings.Fields(htmlAttr.Val) { - if p.requireSandboxOnIFrame[val] { - if !cleanValsSet[val] { - cleanVals = append(cleanVals, val) - cleanValsSet[val] = true - } - } - } - cleanAttrs[i].Val = strings.Join(cleanVals, " ") - } - } - - if !sandboxFound { - sandbox := html.Attribute{} - sandbox.Key = "sandbox" - sandbox.Val = "" - cleanAttrs = append(cleanAttrs, sandbox) - } - } - - return cleanAttrs -} - -func (p *Policy) sanitizeStyles(attr html.Attribute, elementName string) html.Attribute { - sps := p.elsAndStyles[elementName] - if len(sps) == 0 { - sps = map[string][]stylePolicy{} - // check for any matching elements, if we don't already have a policy found - // if multiple matches are found they will be overwritten, it's best - // to not have overlapping matchers - for regex, policies := range p.elsMatchingAndStyles { - if regex.MatchString(elementName) { - for k, v := range policies { - sps[k] = append(sps[k], v...) - } - } - } - } - - //Add semi-colon to end to fix parsing issue - attr.Val = strings.TrimRight(attr.Val, " ") - if len(attr.Val) > 0 && attr.Val[len(attr.Val)-1] != ';' { - attr.Val = attr.Val + ";" - } - decs, err := parser.ParseDeclarations(attr.Val) - if err != nil { - attr.Val = "" - return attr - } - clean := []string{} - prefixes := []string{"-webkit-", "-moz-", "-ms-", "-o-", "mso-", "-xv-", "-atsc-", "-wap-", "-khtml-", "prince-", "-ah-", "-hp-", "-ro-", "-rim-", "-tc-"} - -decLoop: - for _, dec := range decs { - tempProperty := strings.ToLower(dec.Property) - tempValue := removeUnicode(strings.ToLower(dec.Value)) - for _, i := range prefixes { - tempProperty = strings.TrimPrefix(tempProperty, i) - } - if spl, ok := sps[tempProperty]; ok { - for _, sp := range spl { - if sp.handler != nil { - if sp.handler(tempValue) { - clean = append(clean, dec.Property+": "+dec.Value) - continue decLoop - } - } else if len(sp.enum) > 0 { - if stringInSlice(tempValue, sp.enum) { - clean = append(clean, dec.Property+": "+dec.Value) - continue decLoop - } - } else if sp.regexp != nil { - if sp.regexp.MatchString(tempValue) { - clean = append(clean, dec.Property+": "+dec.Value) - continue decLoop - } - } - } - } - if spl, ok := p.globalStyles[tempProperty]; ok { - for _, sp := range spl { - if sp.handler != nil { - if sp.handler(tempValue) { - clean = append(clean, dec.Property+": "+dec.Value) - continue decLoop - } - } else if len(sp.enum) > 0 { - if stringInSlice(tempValue, sp.enum) { - clean = append(clean, dec.Property+": "+dec.Value) - continue decLoop - } - } else if sp.regexp != nil { - if sp.regexp.MatchString(tempValue) { - clean = append(clean, dec.Property+": "+dec.Value) - continue decLoop - } - } - } - } - } - if len(clean) > 0 { - attr.Val = strings.Join(clean, "; ") - } else { - attr.Val = "" - } - return attr -} - -func (p *Policy) allowNoAttrs(elementName string) bool { - _, ok := p.setOfElementsAllowedWithoutAttrs[elementName] - if !ok { - for _, r := range p.setOfElementsMatchingAllowedWithoutAttrs { - if r.MatchString(elementName) { - ok = true - break - } - } - } - return ok -} - -func (p *Policy) validURL(rawurl string) (string, bool) { - if p.requireParseableURLs { - // URLs are valid if when space is trimmed the URL is valid - rawurl = strings.TrimSpace(rawurl) - - // URLs cannot contain whitespace, unless it is a data-uri - if strings.Contains(rawurl, " ") || - strings.Contains(rawurl, "\t") || - strings.Contains(rawurl, "\n") { - if !strings.HasPrefix(rawurl, `data:`) { - return "", false - } - - // Remove \r and \n from base64 encoded data to pass url.Parse. - matched := dataURIbase64Prefix.FindString(rawurl) - if matched != "" { - rawurl = matched + strings.Replace( - strings.Replace( - rawurl[len(matched):], - "\r", - "", - -1, - ), - "\n", - "", - -1, - ) - } - } - - // URLs are valid if they parse - u, err := url.Parse(rawurl) - if err != nil { - return "", false - } - - if u.Scheme != "" { - urlPolicies, ok := p.allowURLSchemes[u.Scheme] - if !ok { - for _, r := range p.allowURLSchemeRegexps { - if r.MatchString(u.Scheme) { - return u.String(), true - } - } - - return "", false - } - - if len(urlPolicies) == 0 { - return u.String(), true - } - - for _, urlPolicy := range urlPolicies { - if urlPolicy(u) { - return u.String(), true - } - } - - return "", false - } - - if p.allowRelativeURLs { - if u.String() != "" { - return u.String(), true - } - } - - return "", false - } - - return rawurl, true -} - -func linkable(elementName string) bool { - switch elementName { - case "a", "area", "base", "link": - // elements that allow .href - return true - case "blockquote", "del", "ins", "q": - // elements that allow .cite - return true - case "audio", "embed", "iframe", "img", "input", "script", "track", "video": - // elements that allow .src - return true - default: - return false - } -} - -// stringInSlice returns true if needle exists in haystack -func stringInSlice(needle string, haystack []string) bool { - for _, straw := range haystack { - if strings.EqualFold(straw, needle) { - return true - } - } - return false -} - -func isDataAttribute(val string) bool { - if !dataAttribute.MatchString(val) { - return false - } - rest := strings.Split(val, "data-") - if len(rest) == 1 { - return false - } - // data-xml* is invalid. - if dataAttributeXMLPrefix.MatchString(rest[1]) { - return false - } - // no uppercase or semi-colons allowed. - if dataAttributeInvalidChars.MatchString(rest[1]) { - return false - } - return true -} - -func removeUnicode(value string) string { - substitutedValue := value - currentLoc := cssUnicodeChar.FindStringIndex(substitutedValue) - for currentLoc != nil { - - character := substitutedValue[currentLoc[0]+1 : currentLoc[1]] - character = strings.TrimSpace(character) - if len(character) < 4 { - character = strings.Repeat("0", 4-len(character)) + character - } else { - for len(character) > 4 { - if character[0] != '0' { - character = "" - break - } else { - character = character[1:] - } - } - } - character = "\\u" + character - translatedChar, err := strconv.Unquote(`"` + character + `"`) - translatedChar = strings.TrimSpace(translatedChar) - if err != nil { - return "" - } - substitutedValue = substitutedValue[0:currentLoc[0]] + translatedChar + substitutedValue[currentLoc[1]:] - currentLoc = cssUnicodeChar.FindStringIndex(substitutedValue) - } - return substitutedValue -} - -func (p *Policy) matchRegex(elementName string) (map[string][]attrPolicy, bool) { - aps := make(map[string][]attrPolicy, 0) - matched := false - for regex, attrs := range p.elsMatchingAndAttrs { - if regex.MatchString(elementName) { - matched = true - for k, v := range attrs { - aps[k] = append(aps[k], v...) - } - } - } - return aps, matched -} - -// normaliseElementName takes a HTML element like " that closes the next token. If - // non-empty, the subsequent call to Next will return a raw or RCDATA text - // token: one that treats "

" as text instead of an element. - // rawTag's contents are lower-cased. - rawTag string - // textIsRaw is whether the current text token's data is not escaped. - textIsRaw bool - // convertNUL is whether NUL bytes in the current token's data should - // be converted into \ufffd replacement characters. - convertNUL bool - // allowCDATA is whether CDATA sections are allowed in the current context. - allowCDATA bool -} - -// AllowCDATA sets whether or not the tokenizer recognizes as -// the text "foo". The default value is false, which means to recognize it as -// a bogus comment "" instead. -// -// Strictly speaking, an HTML5 compliant tokenizer should allow CDATA if and -// only if tokenizing foreign content, such as MathML and SVG. However, -// tracking foreign-contentness is difficult to do purely in the tokenizer, -// as opposed to the parser, due to HTML integration points: an element -// can contain a that is foreign-to-SVG but not foreign-to- -// HTML. For strict compliance with the HTML5 tokenization algorithm, it is the -// responsibility of the user of a tokenizer to call AllowCDATA as appropriate. -// In practice, if using the tokenizer without caring whether MathML or SVG -// CDATA is text or comments, such as tokenizing HTML to find all the anchor -// text, it is acceptable to ignore this responsibility. -func (z *Tokenizer) AllowCDATA(allowCDATA bool) { - z.allowCDATA = allowCDATA -} - -// NextIsNotRawText instructs the tokenizer that the next token should not be -// considered as 'raw text'. Some elements, such as script and title elements, -// normally require the next token after the opening tag to be 'raw text' that -// has no child elements. For example, tokenizing "a<b>c</b>d" -// yields a start tag token for "", a text token for "a<b>c</b>d", and -// an end tag token for "". There are no distinct start tag or end tag -// tokens for the "" and "". -// -// This tokenizer implementation will generally look for raw text at the right -// times. Strictly speaking, an HTML5 compliant tokenizer should not look for -// raw text if in foreign content: generally needs raw text, but a -// <title> inside an <svg> does not. Another example is that a <textarea> -// generally needs raw text, but a <textarea> is not allowed as an immediate -// child of a <select>; in normal parsing, a <textarea> implies </select>, but -// one cannot close the implicit element when parsing a <select>'s InnerHTML. -// Similarly to AllowCDATA, tracking the correct moment to override raw-text- -// ness is difficult to do purely in the tokenizer, as opposed to the parser. -// For strict compliance with the HTML5 tokenization algorithm, it is the -// responsibility of the user of a tokenizer to call NextIsNotRawText as -// appropriate. In practice, like AllowCDATA, it is acceptable to ignore this -// responsibility for basic usage. -// -// Note that this 'raw text' concept is different from the one offered by the -// Tokenizer.Raw method. -func (z *Tokenizer) NextIsNotRawText() { - z.rawTag = "" -} - -// Err returns the error associated with the most recent ErrorToken token. -// This is typically io.EOF, meaning the end of tokenization. -func (z *Tokenizer) Err() error { - if z.tt != ErrorToken { - return nil - } - return z.err -} - -// readByte returns the next byte from the input stream, doing a buffered read -// from z.r into z.buf if necessary. z.buf[z.raw.start:z.raw.end] remains a contiguous byte -// slice that holds all the bytes read so far for the current token. -// It sets z.err if the underlying reader returns an error. -// Pre-condition: z.err == nil. -func (z *Tokenizer) readByte() byte { - if z.raw.end >= len(z.buf) { - // Our buffer is exhausted and we have to read from z.r. Check if the - // previous read resulted in an error. - if z.readErr != nil { - z.err = z.readErr - return 0 - } - // We copy z.buf[z.raw.start:z.raw.end] to the beginning of z.buf. If the length - // z.raw.end - z.raw.start is more than half the capacity of z.buf, then we - // allocate a new buffer before the copy. - c := cap(z.buf) - d := z.raw.end - z.raw.start - var buf1 []byte - if 2*d > c { - buf1 = make([]byte, d, 2*c) - } else { - buf1 = z.buf[:d] - } - copy(buf1, z.buf[z.raw.start:z.raw.end]) - if x := z.raw.start; x != 0 { - // Adjust the data/attr spans to refer to the same contents after the copy. - z.data.start -= x - z.data.end -= x - z.pendingAttr[0].start -= x - z.pendingAttr[0].end -= x - z.pendingAttr[1].start -= x - z.pendingAttr[1].end -= x - for i := range z.attr { - z.attr[i][0].start -= x - z.attr[i][0].end -= x - z.attr[i][1].start -= x - z.attr[i][1].end -= x - } - } - z.raw.start, z.raw.end, z.buf = 0, d, buf1[:d] - // Now that we have copied the live bytes to the start of the buffer, - // we read from z.r into the remainder. - var n int - n, z.readErr = readAtLeastOneByte(z.r, buf1[d:cap(buf1)]) - if n == 0 { - z.err = z.readErr - return 0 - } - z.buf = buf1[:d+n] - } - x := z.buf[z.raw.end] - z.raw.end++ - if z.maxBuf > 0 && z.raw.end-z.raw.start >= z.maxBuf { - z.err = ErrBufferExceeded - return 0 - } - return x -} - -// Buffered returns a slice containing data buffered but not yet tokenized. -func (z *Tokenizer) Buffered() []byte { - return z.buf[z.raw.end:] -} - -// readAtLeastOneByte wraps an io.Reader so that reading cannot return (0, nil). -// It returns io.ErrNoProgress if the underlying r.Read method returns (0, nil) -// too many times in succession. -func readAtLeastOneByte(r io.Reader, b []byte) (int, error) { - for i := 0; i < 100; i++ { - if n, err := r.Read(b); n != 0 || err != nil { - return n, err - } - } - return 0, io.ErrNoProgress -} - -// skipWhiteSpace skips past any white space. -func (z *Tokenizer) skipWhiteSpace() { - if z.err != nil { - return - } - for { - c := z.readByte() - if z.err != nil { - return - } - switch c { - case ' ', '\n', '\r', '\t', '\f': - // No-op. - default: - z.raw.end-- - return - } - } -} - -// readRawOrRCDATA reads until the next "</foo>", where "foo" is z.rawTag and -// is typically something like "script" or "textarea". -func (z *Tokenizer) readRawOrRCDATA() { - if z.rawTag == "script" { - z.readScript() - z.textIsRaw = true - z.rawTag = "" - return - } -loop: - for { - c := z.readByte() - if z.err != nil { - break loop - } - if c != '<' { - continue loop - } - c = z.readByte() - if z.err != nil { - break loop - } - if c != '/' { - z.raw.end-- - continue loop - } - if z.readRawEndTag() || z.err != nil { - break loop - } - } - z.data.end = z.raw.end - // A textarea's or title's RCDATA can contain escaped entities. - z.textIsRaw = z.rawTag != "textarea" && z.rawTag != "title" - z.rawTag = "" -} - -// readRawEndTag attempts to read a tag like "</foo>", where "foo" is z.rawTag. -// If it succeeds, it backs up the input position to reconsume the tag and -// returns true. Otherwise it returns false. The opening "</" has already been -// consumed. -func (z *Tokenizer) readRawEndTag() bool { - for i := 0; i < len(z.rawTag); i++ { - c := z.readByte() - if z.err != nil { - return false - } - if c != z.rawTag[i] && c != z.rawTag[i]-('a'-'A') { - z.raw.end-- - return false - } - } - c := z.readByte() - if z.err != nil { - return false - } - switch c { - case ' ', '\n', '\r', '\t', '\f', '/', '>': - // The 3 is 2 for the leading "</" plus 1 for the trailing character c. - z.raw.end -= 3 + len(z.rawTag) - return true - } - z.raw.end-- - return false -} - -// readScript reads until the next </script> tag, following the byzantine -// rules for escaping/hiding the closing tag. -func (z *Tokenizer) readScript() { - defer func() { - z.data.end = z.raw.end - }() - var c byte - -scriptData: - c = z.readByte() - if z.err != nil { - return - } - if c == '<' { - goto scriptDataLessThanSign - } - goto scriptData - -scriptDataLessThanSign: - c = z.readByte() - if z.err != nil { - return - } - switch c { - case '/': - goto scriptDataEndTagOpen - case '!': - goto scriptDataEscapeStart - } - z.raw.end-- - goto scriptData - -scriptDataEndTagOpen: - if z.readRawEndTag() || z.err != nil { - return - } - goto scriptData - -scriptDataEscapeStart: - c = z.readByte() - if z.err != nil { - return - } - if c == '-' { - goto scriptDataEscapeStartDash - } - z.raw.end-- - goto scriptData - -scriptDataEscapeStartDash: - c = z.readByte() - if z.err != nil { - return - } - if c == '-' { - goto scriptDataEscapedDashDash - } - z.raw.end-- - goto scriptData - -scriptDataEscaped: - c = z.readByte() - if z.err != nil { - return - } - switch c { - case '-': - goto scriptDataEscapedDash - case '<': - goto scriptDataEscapedLessThanSign - } - goto scriptDataEscaped - -scriptDataEscapedDash: - c = z.readByte() - if z.err != nil { - return - } - switch c { - case '-': - goto scriptDataEscapedDashDash - case '<': - goto scriptDataEscapedLessThanSign - } - goto scriptDataEscaped - -scriptDataEscapedDashDash: - c = z.readByte() - if z.err != nil { - return - } - switch c { - case '-': - goto scriptDataEscapedDashDash - case '<': - goto scriptDataEscapedLessThanSign - case '>': - goto scriptData - } - goto scriptDataEscaped - -scriptDataEscapedLessThanSign: - c = z.readByte() - if z.err != nil { - return - } - if c == '/' { - goto scriptDataEscapedEndTagOpen - } - if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' { - goto scriptDataDoubleEscapeStart - } - z.raw.end-- - goto scriptData - -scriptDataEscapedEndTagOpen: - if z.readRawEndTag() || z.err != nil { - return - } - goto scriptDataEscaped - -scriptDataDoubleEscapeStart: - z.raw.end-- - for i := 0; i < len("script"); i++ { - c = z.readByte() - if z.err != nil { - return - } - if c != "script"[i] && c != "SCRIPT"[i] { - z.raw.end-- - goto scriptDataEscaped - } - } - c = z.readByte() - if z.err != nil { - return - } - switch c { - case ' ', '\n', '\r', '\t', '\f', '/', '>': - goto scriptDataDoubleEscaped - } - z.raw.end-- - goto scriptDataEscaped - -scriptDataDoubleEscaped: - c = z.readByte() - if z.err != nil { - return - } - switch c { - case '-': - goto scriptDataDoubleEscapedDash - case '<': - goto scriptDataDoubleEscapedLessThanSign - } - goto scriptDataDoubleEscaped - -scriptDataDoubleEscapedDash: - c = z.readByte() - if z.err != nil { - return - } - switch c { - case '-': - goto scriptDataDoubleEscapedDashDash - case '<': - goto scriptDataDoubleEscapedLessThanSign - } - goto scriptDataDoubleEscaped - -scriptDataDoubleEscapedDashDash: - c = z.readByte() - if z.err != nil { - return - } - switch c { - case '-': - goto scriptDataDoubleEscapedDashDash - case '<': - goto scriptDataDoubleEscapedLessThanSign - case '>': - goto scriptData - } - goto scriptDataDoubleEscaped - -scriptDataDoubleEscapedLessThanSign: - c = z.readByte() - if z.err != nil { - return - } - if c == '/' { - goto scriptDataDoubleEscapeEnd - } - z.raw.end-- - goto scriptDataDoubleEscaped - -scriptDataDoubleEscapeEnd: - if z.readRawEndTag() { - z.raw.end += len("</script>") - goto scriptDataEscaped - } - if z.err != nil { - return - } - goto scriptDataDoubleEscaped -} - -// readComment reads the next comment token starting with "<!--". The opening -// "<!--" has already been consumed. -func (z *Tokenizer) readComment() { - // When modifying this function, consider manually increasing the - // maxSuffixLen constant in func TestComments, from 6 to e.g. 9 or more. - // That increase should only be temporary, not committed, as it - // exponentially affects the test running time. - - z.data.start = z.raw.end - defer func() { - if z.data.end < z.data.start { - // It's a comment with no data, like <!-->. - z.data.end = z.data.start - } - }() - - var dashCount int - beginning := true - for { - c := z.readByte() - if z.err != nil { - z.data.end = z.calculateAbruptCommentDataEnd() - return - } - switch c { - case '-': - dashCount++ - continue - case '>': - if dashCount >= 2 || beginning { - z.data.end = z.raw.end - len("-->") - return - } - case '!': - if dashCount >= 2 { - c = z.readByte() - if z.err != nil { - z.data.end = z.calculateAbruptCommentDataEnd() - return - } else if c == '>' { - z.data.end = z.raw.end - len("--!>") - return - } else if c == '-' { - dashCount = 1 - beginning = false - continue - } - } - } - dashCount = 0 - beginning = false - } -} - -func (z *Tokenizer) calculateAbruptCommentDataEnd() int { - raw := z.Raw() - const prefixLen = len("<!--") - if len(raw) >= prefixLen { - raw = raw[prefixLen:] - if hasSuffix(raw, "--!") { - return z.raw.end - 3 - } else if hasSuffix(raw, "--") { - return z.raw.end - 2 - } else if hasSuffix(raw, "-") { - return z.raw.end - 1 - } - } - return z.raw.end -} - -func hasSuffix(b []byte, suffix string) bool { - if len(b) < len(suffix) { - return false - } - b = b[len(b)-len(suffix):] - for i := range b { - if b[i] != suffix[i] { - return false - } - } - return true -} - -// readUntilCloseAngle reads until the next ">". -func (z *Tokenizer) readUntilCloseAngle() { - z.data.start = z.raw.end - for { - c := z.readByte() - if z.err != nil { - z.data.end = z.raw.end - return - } - if c == '>' { - z.data.end = z.raw.end - len(">") - return - } - } -} - -// readMarkupDeclaration reads the next token starting with "<!". It might be -// a "<!--comment-->", a "<!DOCTYPE foo>", a "<![CDATA[section]]>" or -// "<!a bogus comment". The opening "<!" has already been consumed. -func (z *Tokenizer) readMarkupDeclaration() TokenType { - z.data.start = z.raw.end - var c [2]byte - for i := 0; i < 2; i++ { - c[i] = z.readByte() - if z.err != nil { - z.data.end = z.raw.end - return CommentToken - } - } - if c[0] == '-' && c[1] == '-' { - z.readComment() - return CommentToken - } - z.raw.end -= 2 - if z.readDoctype() { - return DoctypeToken - } - if z.allowCDATA && z.readCDATA() { - z.convertNUL = true - return TextToken - } - // It's a bogus comment. - z.readUntilCloseAngle() - return CommentToken -} - -// readDoctype attempts to read a doctype declaration and returns true if -// successful. The opening "<!" has already been consumed. -func (z *Tokenizer) readDoctype() bool { - const s = "DOCTYPE" - for i := 0; i < len(s); i++ { - c := z.readByte() - if z.err != nil { - z.data.end = z.raw.end - return false - } - if c != s[i] && c != s[i]+('a'-'A') { - // Back up to read the fragment of "DOCTYPE" again. - z.raw.end = z.data.start - return false - } - } - if z.skipWhiteSpace(); z.err != nil { - z.data.start = z.raw.end - z.data.end = z.raw.end - return true - } - z.readUntilCloseAngle() - return true -} - -// readCDATA attempts to read a CDATA section and returns true if -// successful. The opening "<!" has already been consumed. -func (z *Tokenizer) readCDATA() bool { - const s = "[CDATA[" - for i := 0; i < len(s); i++ { - c := z.readByte() - if z.err != nil { - z.data.end = z.raw.end - return false - } - if c != s[i] { - // Back up to read the fragment of "[CDATA[" again. - z.raw.end = z.data.start - return false - } - } - z.data.start = z.raw.end - brackets := 0 - for { - c := z.readByte() - if z.err != nil { - z.data.end = z.raw.end - return true - } - switch c { - case ']': - brackets++ - case '>': - if brackets >= 2 { - z.data.end = z.raw.end - len("]]>") - return true - } - brackets = 0 - default: - brackets = 0 - } - } -} - -// startTagIn returns whether the start tag in z.buf[z.data.start:z.data.end] -// case-insensitively matches any element of ss. -func (z *Tokenizer) startTagIn(ss ...string) bool { -loop: - for _, s := range ss { - if z.data.end-z.data.start != len(s) { - continue loop - } - for i := 0; i < len(s); i++ { - c := z.buf[z.data.start+i] - if 'A' <= c && c <= 'Z' { - c += 'a' - 'A' - } - if c != s[i] { - continue loop - } - } - return true - } - return false -} - -// readStartTag reads the next start tag token. The opening "<a" has already -// been consumed, where 'a' means anything in [A-Za-z]. -func (z *Tokenizer) readStartTag() TokenType { - z.readTag(true) - if z.err != nil { - return ErrorToken - } - // Several tags flag the tokenizer's next token as raw. - c, raw := z.buf[z.data.start], false - if 'A' <= c && c <= 'Z' { - c += 'a' - 'A' - } - switch c { - case 'i': - raw = z.startTagIn("iframe") - case 'n': - raw = z.startTagIn("noembed", "noframes", "noscript") - case 'p': - raw = z.startTagIn("plaintext") - case 's': - raw = z.startTagIn("script", "style") - case 't': - raw = z.startTagIn("textarea", "title") - case 'x': - raw = z.startTagIn("xmp") - } - if raw { - z.rawTag = strings.ToLower(string(z.buf[z.data.start:z.data.end])) - } - // Look for a self-closing token like "<br/>". - if z.err == nil && z.buf[z.raw.end-2] == '/' { - return SelfClosingTagToken - } - return StartTagToken -} - -// readTag reads the next tag token and its attributes. If saveAttr, those -// attributes are saved in z.attr, otherwise z.attr is set to an empty slice. -// The opening "<a" or "</a" has already been consumed, where 'a' means anything -// in [A-Za-z]. -func (z *Tokenizer) readTag(saveAttr bool) { - z.attr = z.attr[:0] - z.nAttrReturned = 0 - // Read the tag name and attribute key/value pairs. - z.readTagName() - if z.skipWhiteSpace(); z.err != nil { - return - } - for { - c := z.readByte() - if z.err != nil || c == '>' { - break - } - z.raw.end-- - z.readTagAttrKey() - z.readTagAttrVal() - // Save pendingAttr if saveAttr and that attribute has a non-empty key. - if saveAttr && z.pendingAttr[0].start != z.pendingAttr[0].end { - z.attr = append(z.attr, z.pendingAttr) - } - if z.skipWhiteSpace(); z.err != nil { - break - } - } -} - -// readTagName sets z.data to the "div" in "<div k=v>". The reader (z.raw.end) -// is positioned such that the first byte of the tag name (the "d" in "<div") -// has already been consumed. -func (z *Tokenizer) readTagName() { - z.data.start = z.raw.end - 1 - for { - c := z.readByte() - if z.err != nil { - z.data.end = z.raw.end - return - } - switch c { - case ' ', '\n', '\r', '\t', '\f': - z.data.end = z.raw.end - 1 - return - case '/', '>': - z.raw.end-- - z.data.end = z.raw.end - return - } - } -} - -// readTagAttrKey sets z.pendingAttr[0] to the "k" in "<div k=v>". -// Precondition: z.err == nil. -func (z *Tokenizer) readTagAttrKey() { - z.pendingAttr[0].start = z.raw.end - for { - c := z.readByte() - if z.err != nil { - z.pendingAttr[0].end = z.raw.end - return - } - switch c { - case ' ', '\n', '\r', '\t', '\f', '/': - z.pendingAttr[0].end = z.raw.end - 1 - return - case '=': - if z.pendingAttr[0].start+1 == z.raw.end { - // WHATWG 13.2.5.32, if we see an equals sign before the attribute name - // begins, we treat it as a character in the attribute name and continue. - continue - } - fallthrough - case '>': - z.raw.end-- - z.pendingAttr[0].end = z.raw.end - return - } - } -} - -// readTagAttrVal sets z.pendingAttr[1] to the "v" in "<div k=v>". -func (z *Tokenizer) readTagAttrVal() { - z.pendingAttr[1].start = z.raw.end - z.pendingAttr[1].end = z.raw.end - if z.skipWhiteSpace(); z.err != nil { - return - } - c := z.readByte() - if z.err != nil { - return - } - if c != '=' { - z.raw.end-- - return - } - if z.skipWhiteSpace(); z.err != nil { - return - } - quote := z.readByte() - if z.err != nil { - return - } - switch quote { - case '>': - z.raw.end-- - return - - case '\'', '"': - z.pendingAttr[1].start = z.raw.end - for { - c := z.readByte() - if z.err != nil { - z.pendingAttr[1].end = z.raw.end - return - } - if c == quote { - z.pendingAttr[1].end = z.raw.end - 1 - return - } - } - - default: - z.pendingAttr[1].start = z.raw.end - 1 - for { - c := z.readByte() - if z.err != nil { - z.pendingAttr[1].end = z.raw.end - return - } - switch c { - case ' ', '\n', '\r', '\t', '\f': - z.pendingAttr[1].end = z.raw.end - 1 - return - case '>': - z.raw.end-- - z.pendingAttr[1].end = z.raw.end - return - } - } - } -} - -// Next scans the next token and returns its type. -func (z *Tokenizer) Next() TokenType { - z.raw.start = z.raw.end - z.data.start = z.raw.end - z.data.end = z.raw.end - if z.err != nil { - z.tt = ErrorToken - return z.tt - } - if z.rawTag != "" { - if z.rawTag == "plaintext" { - // Read everything up to EOF. - for z.err == nil { - z.readByte() - } - z.data.end = z.raw.end - z.textIsRaw = true - } else { - z.readRawOrRCDATA() - } - if z.data.end > z.data.start { - z.tt = TextToken - z.convertNUL = true - return z.tt - } - } - z.textIsRaw = false - z.convertNUL = false - -loop: - for { - c := z.readByte() - if z.err != nil { - break loop - } - if c != '<' { - continue loop - } - - // Check if the '<' we have just read is part of a tag, comment - // or doctype. If not, it's part of the accumulated text token. - c = z.readByte() - if z.err != nil { - break loop - } - var tokenType TokenType - switch { - case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z': - tokenType = StartTagToken - case c == '/': - tokenType = EndTagToken - case c == '!' || c == '?': - // We use CommentToken to mean any of "<!--actual comments-->", - // "<!DOCTYPE declarations>" and "<?xml processing instructions?>". - tokenType = CommentToken - default: - // Reconsume the current character. - z.raw.end-- - continue - } - - // We have a non-text token, but we might have accumulated some text - // before that. If so, we return the text first, and return the non- - // text token on the subsequent call to Next. - if x := z.raw.end - len("<a"); z.raw.start < x { - z.raw.end = x - z.data.end = x - z.tt = TextToken - return z.tt - } - switch tokenType { - case StartTagToken: - z.tt = z.readStartTag() - return z.tt - case EndTagToken: - c = z.readByte() - if z.err != nil { - break loop - } - if c == '>' { - // "</>" does not generate a token at all. Generate an empty comment - // to allow passthrough clients to pick up the data using Raw. - // Reset the tokenizer state and start again. - z.tt = CommentToken - return z.tt - } - if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' { - z.readTag(false) - if z.err != nil { - z.tt = ErrorToken - } else { - z.tt = EndTagToken - } - return z.tt - } - z.raw.end-- - z.readUntilCloseAngle() - z.tt = CommentToken - return z.tt - case CommentToken: - if c == '!' { - z.tt = z.readMarkupDeclaration() - return z.tt - } - z.raw.end-- - z.readUntilCloseAngle() - z.tt = CommentToken - return z.tt - } - } - if z.raw.start < z.raw.end { - z.data.end = z.raw.end - z.tt = TextToken - return z.tt - } - z.tt = ErrorToken - return z.tt -} - -// Raw returns the unmodified text of the current token. Calling Next, Token, -// Text, TagName or TagAttr may change the contents of the returned slice. -// -// The token stream's raw bytes partition the byte stream (up until an -// ErrorToken). There are no overlaps or gaps between two consecutive token's -// raw bytes. One implication is that the byte offset of the current token is -// the sum of the lengths of all previous tokens' raw bytes. -func (z *Tokenizer) Raw() []byte { - return z.buf[z.raw.start:z.raw.end] -} - -// convertNewlines converts "\r" and "\r\n" in s to "\n". -// The conversion happens in place, but the resulting slice may be shorter. -func convertNewlines(s []byte) []byte { - for i, c := range s { - if c != '\r' { - continue - } - - src := i + 1 - if src >= len(s) || s[src] != '\n' { - s[i] = '\n' - continue - } - - dst := i - for src < len(s) { - if s[src] == '\r' { - if src+1 < len(s) && s[src+1] == '\n' { - src++ - } - s[dst] = '\n' - } else { - s[dst] = s[src] - } - src++ - dst++ - } - return s[:dst] - } - return s -} - -var ( - nul = []byte("\x00") - replacement = []byte("\ufffd") -) - -// Text returns the unescaped text of a text, comment or doctype token. The -// contents of the returned slice may change on the next call to Next. -func (z *Tokenizer) Text() []byte { - switch z.tt { - case TextToken, CommentToken, DoctypeToken: - s := z.buf[z.data.start:z.data.end] - z.data.start = z.raw.end - z.data.end = z.raw.end - s = convertNewlines(s) - if (z.convertNUL || z.tt == CommentToken) && bytes.Contains(s, nul) { - s = bytes.Replace(s, nul, replacement, -1) - } - if !z.textIsRaw { - s = unescape(s, false) - } - return s - } - return nil -} - -// TagName returns the lower-cased name of a tag token (the `img` out of -// `<IMG SRC="foo">`) and whether the tag has attributes. -// The contents of the returned slice may change on the next call to Next. -func (z *Tokenizer) TagName() (name []byte, hasAttr bool) { - if z.data.start < z.data.end { - switch z.tt { - case StartTagToken, EndTagToken, SelfClosingTagToken: - s := z.buf[z.data.start:z.data.end] - z.data.start = z.raw.end - z.data.end = z.raw.end - return lower(s), z.nAttrReturned < len(z.attr) - } - } - return nil, false -} - -// TagAttr returns the lower-cased key and unescaped value of the next unparsed -// attribute for the current tag token and whether there are more attributes. -// The contents of the returned slices may change on the next call to Next. -func (z *Tokenizer) TagAttr() (key, val []byte, moreAttr bool) { - if z.nAttrReturned < len(z.attr) { - switch z.tt { - case StartTagToken, SelfClosingTagToken: - x := z.attr[z.nAttrReturned] - z.nAttrReturned++ - key = z.buf[x[0].start:x[0].end] - val = z.buf[x[1].start:x[1].end] - return lower(key), unescape(convertNewlines(val), true), z.nAttrReturned < len(z.attr) - } - } - return nil, nil, false -} - -// Token returns the current Token. The result's Data and Attr values remain -// valid after subsequent Next calls. -func (z *Tokenizer) Token() Token { - t := Token{Type: z.tt} - switch z.tt { - case TextToken, CommentToken, DoctypeToken: - t.Data = string(z.Text()) - case StartTagToken, SelfClosingTagToken, EndTagToken: - name, moreAttr := z.TagName() - for moreAttr { - var key, val []byte - key, val, moreAttr = z.TagAttr() - t.Attr = append(t.Attr, Attribute{"", atom.String(key), string(val)}) - } - if a := atom.Lookup(name); a != 0 { - t.DataAtom, t.Data = a, a.String() - } else { - t.DataAtom, t.Data = 0, string(name) - } - } - return t -} - -// SetMaxBuf sets a limit on the amount of data buffered during tokenization. -// A value of 0 means unlimited. -func (z *Tokenizer) SetMaxBuf(n int) { - z.maxBuf = n -} - -// NewTokenizer returns a new HTML Tokenizer for the given Reader. -// The input is assumed to be UTF-8 encoded. -func NewTokenizer(r io.Reader) *Tokenizer { - return NewTokenizerFragment(r, "") -} - -// NewTokenizerFragment returns a new HTML Tokenizer for the given Reader, for -// tokenizing an existing element's InnerHTML fragment. contextTag is that -// element's tag, such as "div" or "iframe". -// -// For example, how the InnerHTML "a<b" is tokenized depends on whether it is -// for a <p> tag or a <script> tag. -// -// The input is assumed to be UTF-8 encoded. -func NewTokenizerFragment(r io.Reader, contextTag string) *Tokenizer { - z := &Tokenizer{ - r: r, - buf: make([]byte, 0, 4096), - } - if contextTag != "" { - switch s := strings.ToLower(contextTag); s { - case "iframe", "noembed", "noframes", "noscript", "plaintext", "script", "style", "title", "textarea", "xmp": - z.rawTag = s - } - } - return z -} diff --git a/vendor/golang.org/x/net/internal/socks/client.go b/vendor/golang.org/x/net/internal/socks/client.go deleted file mode 100644 index 3d6f516a..00000000 --- a/vendor/golang.org/x/net/internal/socks/client.go +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package socks - -import ( - "context" - "errors" - "io" - "net" - "strconv" - "time" -) - -var ( - noDeadline = time.Time{} - aLongTimeAgo = time.Unix(1, 0) -) - -func (d *Dialer) connect(ctx context.Context, c net.Conn, address string) (_ net.Addr, ctxErr error) { - host, port, err := splitHostPort(address) - if err != nil { - return nil, err - } - if deadline, ok := ctx.Deadline(); ok && !deadline.IsZero() { - c.SetDeadline(deadline) - defer c.SetDeadline(noDeadline) - } - if ctx != context.Background() { - errCh := make(chan error, 1) - done := make(chan struct{}) - defer func() { - close(done) - if ctxErr == nil { - ctxErr = <-errCh - } - }() - go func() { - select { - case <-ctx.Done(): - c.SetDeadline(aLongTimeAgo) - errCh <- ctx.Err() - case <-done: - errCh <- nil - } - }() - } - - b := make([]byte, 0, 6+len(host)) // the size here is just an estimate - b = append(b, Version5) - if len(d.AuthMethods) == 0 || d.Authenticate == nil { - b = append(b, 1, byte(AuthMethodNotRequired)) - } else { - ams := d.AuthMethods - if len(ams) > 255 { - return nil, errors.New("too many authentication methods") - } - b = append(b, byte(len(ams))) - for _, am := range ams { - b = append(b, byte(am)) - } - } - if _, ctxErr = c.Write(b); ctxErr != nil { - return - } - - if _, ctxErr = io.ReadFull(c, b[:2]); ctxErr != nil { - return - } - if b[0] != Version5 { - return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0]))) - } - am := AuthMethod(b[1]) - if am == AuthMethodNoAcceptableMethods { - return nil, errors.New("no acceptable authentication methods") - } - if d.Authenticate != nil { - if ctxErr = d.Authenticate(ctx, c, am); ctxErr != nil { - return - } - } - - b = b[:0] - b = append(b, Version5, byte(d.cmd), 0) - if ip := net.ParseIP(host); ip != nil { - if ip4 := ip.To4(); ip4 != nil { - b = append(b, AddrTypeIPv4) - b = append(b, ip4...) - } else if ip6 := ip.To16(); ip6 != nil { - b = append(b, AddrTypeIPv6) - b = append(b, ip6...) - } else { - return nil, errors.New("unknown address type") - } - } else { - if len(host) > 255 { - return nil, errors.New("FQDN too long") - } - b = append(b, AddrTypeFQDN) - b = append(b, byte(len(host))) - b = append(b, host...) - } - b = append(b, byte(port>>8), byte(port)) - if _, ctxErr = c.Write(b); ctxErr != nil { - return - } - - if _, ctxErr = io.ReadFull(c, b[:4]); ctxErr != nil { - return - } - if b[0] != Version5 { - return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0]))) - } - if cmdErr := Reply(b[1]); cmdErr != StatusSucceeded { - return nil, errors.New("unknown error " + cmdErr.String()) - } - if b[2] != 0 { - return nil, errors.New("non-zero reserved field") - } - l := 2 - var a Addr - switch b[3] { - case AddrTypeIPv4: - l += net.IPv4len - a.IP = make(net.IP, net.IPv4len) - case AddrTypeIPv6: - l += net.IPv6len - a.IP = make(net.IP, net.IPv6len) - case AddrTypeFQDN: - if _, err := io.ReadFull(c, b[:1]); err != nil { - return nil, err - } - l += int(b[0]) - default: - return nil, errors.New("unknown address type " + strconv.Itoa(int(b[3]))) - } - if cap(b) < l { - b = make([]byte, l) - } else { - b = b[:l] - } - if _, ctxErr = io.ReadFull(c, b); ctxErr != nil { - return - } - if a.IP != nil { - copy(a.IP, b) - } else { - a.Name = string(b[:len(b)-2]) - } - a.Port = int(b[len(b)-2])<<8 | int(b[len(b)-1]) - return &a, nil -} - -func splitHostPort(address string) (string, int, error) { - host, port, err := net.SplitHostPort(address) - if err != nil { - return "", 0, err - } - portnum, err := strconv.Atoi(port) - if err != nil { - return "", 0, err - } - if 1 > portnum || portnum > 0xffff { - return "", 0, errors.New("port number out of range " + port) - } - return host, portnum, nil -} diff --git a/vendor/golang.org/x/net/internal/socks/socks.go b/vendor/golang.org/x/net/internal/socks/socks.go deleted file mode 100644 index 84fcc32b..00000000 --- a/vendor/golang.org/x/net/internal/socks/socks.go +++ /dev/null @@ -1,317 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package socks provides a SOCKS version 5 client implementation. -// -// SOCKS protocol version 5 is defined in RFC 1928. -// Username/Password authentication for SOCKS version 5 is defined in -// RFC 1929. -package socks - -import ( - "context" - "errors" - "io" - "net" - "strconv" -) - -// A Command represents a SOCKS command. -type Command int - -func (cmd Command) String() string { - switch cmd { - case CmdConnect: - return "socks connect" - case cmdBind: - return "socks bind" - default: - return "socks " + strconv.Itoa(int(cmd)) - } -} - -// An AuthMethod represents a SOCKS authentication method. -type AuthMethod int - -// A Reply represents a SOCKS command reply code. -type Reply int - -func (code Reply) String() string { - switch code { - case StatusSucceeded: - return "succeeded" - case 0x01: - return "general SOCKS server failure" - case 0x02: - return "connection not allowed by ruleset" - case 0x03: - return "network unreachable" - case 0x04: - return "host unreachable" - case 0x05: - return "connection refused" - case 0x06: - return "TTL expired" - case 0x07: - return "command not supported" - case 0x08: - return "address type not supported" - default: - return "unknown code: " + strconv.Itoa(int(code)) - } -} - -// Wire protocol constants. -const ( - Version5 = 0x05 - - AddrTypeIPv4 = 0x01 - AddrTypeFQDN = 0x03 - AddrTypeIPv6 = 0x04 - - CmdConnect Command = 0x01 // establishes an active-open forward proxy connection - cmdBind Command = 0x02 // establishes a passive-open forward proxy connection - - AuthMethodNotRequired AuthMethod = 0x00 // no authentication required - AuthMethodUsernamePassword AuthMethod = 0x02 // use username/password - AuthMethodNoAcceptableMethods AuthMethod = 0xff // no acceptable authentication methods - - StatusSucceeded Reply = 0x00 -) - -// An Addr represents a SOCKS-specific address. -// Either Name or IP is used exclusively. -type Addr struct { - Name string // fully-qualified domain name - IP net.IP - Port int -} - -func (a *Addr) Network() string { return "socks" } - -func (a *Addr) String() string { - if a == nil { - return "<nil>" - } - port := strconv.Itoa(a.Port) - if a.IP == nil { - return net.JoinHostPort(a.Name, port) - } - return net.JoinHostPort(a.IP.String(), port) -} - -// A Conn represents a forward proxy connection. -type Conn struct { - net.Conn - - boundAddr net.Addr -} - -// BoundAddr returns the address assigned by the proxy server for -// connecting to the command target address from the proxy server. -func (c *Conn) BoundAddr() net.Addr { - if c == nil { - return nil - } - return c.boundAddr -} - -// A Dialer holds SOCKS-specific options. -type Dialer struct { - cmd Command // either CmdConnect or cmdBind - proxyNetwork string // network between a proxy server and a client - proxyAddress string // proxy server address - - // ProxyDial specifies the optional dial function for - // establishing the transport connection. - ProxyDial func(context.Context, string, string) (net.Conn, error) - - // AuthMethods specifies the list of request authentication - // methods. - // If empty, SOCKS client requests only AuthMethodNotRequired. - AuthMethods []AuthMethod - - // Authenticate specifies the optional authentication - // function. It must be non-nil when AuthMethods is not empty. - // It must return an error when the authentication is failed. - Authenticate func(context.Context, io.ReadWriter, AuthMethod) error -} - -// DialContext connects to the provided address on the provided -// network. -// -// The returned error value may be a net.OpError. When the Op field of -// net.OpError contains "socks", the Source field contains a proxy -// server address and the Addr field contains a command target -// address. -// -// See func Dial of the net package of standard library for a -// description of the network and address parameters. -func (d *Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { - if err := d.validateTarget(network, address); err != nil { - proxy, dst, _ := d.pathAddrs(address) - return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} - } - if ctx == nil { - proxy, dst, _ := d.pathAddrs(address) - return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")} - } - var err error - var c net.Conn - if d.ProxyDial != nil { - c, err = d.ProxyDial(ctx, d.proxyNetwork, d.proxyAddress) - } else { - var dd net.Dialer - c, err = dd.DialContext(ctx, d.proxyNetwork, d.proxyAddress) - } - if err != nil { - proxy, dst, _ := d.pathAddrs(address) - return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} - } - a, err := d.connect(ctx, c, address) - if err != nil { - c.Close() - proxy, dst, _ := d.pathAddrs(address) - return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} - } - return &Conn{Conn: c, boundAddr: a}, nil -} - -// DialWithConn initiates a connection from SOCKS server to the target -// network and address using the connection c that is already -// connected to the SOCKS server. -// -// It returns the connection's local address assigned by the SOCKS -// server. -func (d *Dialer) DialWithConn(ctx context.Context, c net.Conn, network, address string) (net.Addr, error) { - if err := d.validateTarget(network, address); err != nil { - proxy, dst, _ := d.pathAddrs(address) - return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} - } - if ctx == nil { - proxy, dst, _ := d.pathAddrs(address) - return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")} - } - a, err := d.connect(ctx, c, address) - if err != nil { - proxy, dst, _ := d.pathAddrs(address) - return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} - } - return a, nil -} - -// Dial connects to the provided address on the provided network. -// -// Unlike DialContext, it returns a raw transport connection instead -// of a forward proxy connection. -// -// Deprecated: Use DialContext or DialWithConn instead. -func (d *Dialer) Dial(network, address string) (net.Conn, error) { - if err := d.validateTarget(network, address); err != nil { - proxy, dst, _ := d.pathAddrs(address) - return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} - } - var err error - var c net.Conn - if d.ProxyDial != nil { - c, err = d.ProxyDial(context.Background(), d.proxyNetwork, d.proxyAddress) - } else { - c, err = net.Dial(d.proxyNetwork, d.proxyAddress) - } - if err != nil { - proxy, dst, _ := d.pathAddrs(address) - return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} - } - if _, err := d.DialWithConn(context.Background(), c, network, address); err != nil { - c.Close() - return nil, err - } - return c, nil -} - -func (d *Dialer) validateTarget(network, address string) error { - switch network { - case "tcp", "tcp6", "tcp4": - default: - return errors.New("network not implemented") - } - switch d.cmd { - case CmdConnect, cmdBind: - default: - return errors.New("command not implemented") - } - return nil -} - -func (d *Dialer) pathAddrs(address string) (proxy, dst net.Addr, err error) { - for i, s := range []string{d.proxyAddress, address} { - host, port, err := splitHostPort(s) - if err != nil { - return nil, nil, err - } - a := &Addr{Port: port} - a.IP = net.ParseIP(host) - if a.IP == nil { - a.Name = host - } - if i == 0 { - proxy = a - } else { - dst = a - } - } - return -} - -// NewDialer returns a new Dialer that dials through the provided -// proxy server's network and address. -func NewDialer(network, address string) *Dialer { - return &Dialer{proxyNetwork: network, proxyAddress: address, cmd: CmdConnect} -} - -const ( - authUsernamePasswordVersion = 0x01 - authStatusSucceeded = 0x00 -) - -// UsernamePassword are the credentials for the username/password -// authentication method. -type UsernamePassword struct { - Username string - Password string -} - -// Authenticate authenticates a pair of username and password with the -// proxy server. -func (up *UsernamePassword) Authenticate(ctx context.Context, rw io.ReadWriter, auth AuthMethod) error { - switch auth { - case AuthMethodNotRequired: - return nil - case AuthMethodUsernamePassword: - if len(up.Username) == 0 || len(up.Username) > 255 || len(up.Password) > 255 { - return errors.New("invalid username/password") - } - b := []byte{authUsernamePasswordVersion} - b = append(b, byte(len(up.Username))) - b = append(b, up.Username...) - b = append(b, byte(len(up.Password))) - b = append(b, up.Password...) - // TODO(mikio): handle IO deadlines and cancelation if - // necessary - if _, err := rw.Write(b); err != nil { - return err - } - if _, err := io.ReadFull(rw, b[:2]); err != nil { - return err - } - if b[0] != authUsernamePasswordVersion { - return errors.New("invalid username/password version") - } - if b[1] != authStatusSucceeded { - return errors.New("username/password authentication failed") - } - return nil - } - return errors.New("unsupported authentication method " + strconv.Itoa(int(auth))) -} diff --git a/vendor/golang.org/x/net/proxy/dial.go b/vendor/golang.org/x/net/proxy/dial.go deleted file mode 100644 index 811c2e4e..00000000 --- a/vendor/golang.org/x/net/proxy/dial.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2019 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package proxy - -import ( - "context" - "net" -) - -// A ContextDialer dials using a context. -type ContextDialer interface { - DialContext(ctx context.Context, network, address string) (net.Conn, error) -} - -// Dial works like DialContext on net.Dialer but using a dialer returned by FromEnvironment. -// -// The passed ctx is only used for returning the Conn, not the lifetime of the Conn. -// -// Custom dialers (registered via RegisterDialerType) that do not implement ContextDialer -// can leak a goroutine for as long as it takes the underlying Dialer implementation to timeout. -// -// A Conn returned from a successful Dial after the context has been cancelled will be immediately closed. -func Dial(ctx context.Context, network, address string) (net.Conn, error) { - d := FromEnvironment() - if xd, ok := d.(ContextDialer); ok { - return xd.DialContext(ctx, network, address) - } - return dialContext(ctx, d, network, address) -} - -// WARNING: this can leak a goroutine for as long as the underlying Dialer implementation takes to timeout -// A Conn returned from a successful Dial after the context has been cancelled will be immediately closed. -func dialContext(ctx context.Context, d Dialer, network, address string) (net.Conn, error) { - var ( - conn net.Conn - done = make(chan struct{}, 1) - err error - ) - go func() { - conn, err = d.Dial(network, address) - close(done) - if conn != nil && ctx.Err() != nil { - conn.Close() - } - }() - select { - case <-ctx.Done(): - err = ctx.Err() - case <-done: - } - return conn, err -} diff --git a/vendor/golang.org/x/net/proxy/direct.go b/vendor/golang.org/x/net/proxy/direct.go deleted file mode 100644 index 3d66bdef..00000000 --- a/vendor/golang.org/x/net/proxy/direct.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package proxy - -import ( - "context" - "net" -) - -type direct struct{} - -// Direct implements Dialer by making network connections directly using net.Dial or net.DialContext. -var Direct = direct{} - -var ( - _ Dialer = Direct - _ ContextDialer = Direct -) - -// Dial directly invokes net.Dial with the supplied parameters. -func (direct) Dial(network, addr string) (net.Conn, error) { - return net.Dial(network, addr) -} - -// DialContext instantiates a net.Dialer and invokes its DialContext receiver with the supplied parameters. -func (direct) DialContext(ctx context.Context, network, addr string) (net.Conn, error) { - var d net.Dialer - return d.DialContext(ctx, network, addr) -} diff --git a/vendor/golang.org/x/net/proxy/per_host.go b/vendor/golang.org/x/net/proxy/per_host.go deleted file mode 100644 index 573fe79e..00000000 --- a/vendor/golang.org/x/net/proxy/per_host.go +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package proxy - -import ( - "context" - "net" - "strings" -) - -// A PerHost directs connections to a default Dialer unless the host name -// requested matches one of a number of exceptions. -type PerHost struct { - def, bypass Dialer - - bypassNetworks []*net.IPNet - bypassIPs []net.IP - bypassZones []string - bypassHosts []string -} - -// NewPerHost returns a PerHost Dialer that directs connections to either -// defaultDialer or bypass, depending on whether the connection matches one of -// the configured rules. -func NewPerHost(defaultDialer, bypass Dialer) *PerHost { - return &PerHost{ - def: defaultDialer, - bypass: bypass, - } -} - -// Dial connects to the address addr on the given network through either -// defaultDialer or bypass. -func (p *PerHost) Dial(network, addr string) (c net.Conn, err error) { - host, _, err := net.SplitHostPort(addr) - if err != nil { - return nil, err - } - - return p.dialerForRequest(host).Dial(network, addr) -} - -// DialContext connects to the address addr on the given network through either -// defaultDialer or bypass. -func (p *PerHost) DialContext(ctx context.Context, network, addr string) (c net.Conn, err error) { - host, _, err := net.SplitHostPort(addr) - if err != nil { - return nil, err - } - d := p.dialerForRequest(host) - if x, ok := d.(ContextDialer); ok { - return x.DialContext(ctx, network, addr) - } - return dialContext(ctx, d, network, addr) -} - -func (p *PerHost) dialerForRequest(host string) Dialer { - if ip := net.ParseIP(host); ip != nil { - for _, net := range p.bypassNetworks { - if net.Contains(ip) { - return p.bypass - } - } - for _, bypassIP := range p.bypassIPs { - if bypassIP.Equal(ip) { - return p.bypass - } - } - return p.def - } - - for _, zone := range p.bypassZones { - if strings.HasSuffix(host, zone) { - return p.bypass - } - if host == zone[1:] { - // For a zone ".example.com", we match "example.com" - // too. - return p.bypass - } - } - for _, bypassHost := range p.bypassHosts { - if bypassHost == host { - return p.bypass - } - } - return p.def -} - -// AddFromString parses a string that contains comma-separated values -// specifying hosts that should use the bypass proxy. Each value is either an -// IP address, a CIDR range, a zone (*.example.com) or a host name -// (localhost). A best effort is made to parse the string and errors are -// ignored. -func (p *PerHost) AddFromString(s string) { - hosts := strings.Split(s, ",") - for _, host := range hosts { - host = strings.TrimSpace(host) - if len(host) == 0 { - continue - } - if strings.Contains(host, "/") { - // We assume that it's a CIDR address like 127.0.0.0/8 - if _, net, err := net.ParseCIDR(host); err == nil { - p.AddNetwork(net) - } - continue - } - if ip := net.ParseIP(host); ip != nil { - p.AddIP(ip) - continue - } - if strings.HasPrefix(host, "*.") { - p.AddZone(host[1:]) - continue - } - p.AddHost(host) - } -} - -// AddIP specifies an IP address that will use the bypass proxy. Note that -// this will only take effect if a literal IP address is dialed. A connection -// to a named host will never match an IP. -func (p *PerHost) AddIP(ip net.IP) { - p.bypassIPs = append(p.bypassIPs, ip) -} - -// AddNetwork specifies an IP range that will use the bypass proxy. Note that -// this will only take effect if a literal IP address is dialed. A connection -// to a named host will never match. -func (p *PerHost) AddNetwork(net *net.IPNet) { - p.bypassNetworks = append(p.bypassNetworks, net) -} - -// AddZone specifies a DNS suffix that will use the bypass proxy. A zone of -// "example.com" matches "example.com" and all of its subdomains. -func (p *PerHost) AddZone(zone string) { - if strings.HasSuffix(zone, ".") { - zone = zone[:len(zone)-1] - } - if !strings.HasPrefix(zone, ".") { - zone = "." + zone - } - p.bypassZones = append(p.bypassZones, zone) -} - -// AddHost specifies a host name that will use the bypass proxy. -func (p *PerHost) AddHost(host string) { - if strings.HasSuffix(host, ".") { - host = host[:len(host)-1] - } - p.bypassHosts = append(p.bypassHosts, host) -} diff --git a/vendor/golang.org/x/net/proxy/proxy.go b/vendor/golang.org/x/net/proxy/proxy.go deleted file mode 100644 index 9ff4b9a7..00000000 --- a/vendor/golang.org/x/net/proxy/proxy.go +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package proxy provides support for a variety of protocols to proxy network -// data. -package proxy // import "golang.org/x/net/proxy" - -import ( - "errors" - "net" - "net/url" - "os" - "sync" -) - -// A Dialer is a means to establish a connection. -// Custom dialers should also implement ContextDialer. -type Dialer interface { - // Dial connects to the given address via the proxy. - Dial(network, addr string) (c net.Conn, err error) -} - -// Auth contains authentication parameters that specific Dialers may require. -type Auth struct { - User, Password string -} - -// FromEnvironment returns the dialer specified by the proxy-related -// variables in the environment and makes underlying connections -// directly. -func FromEnvironment() Dialer { - return FromEnvironmentUsing(Direct) -} - -// FromEnvironmentUsing returns the dialer specify by the proxy-related -// variables in the environment and makes underlying connections -// using the provided forwarding Dialer (for instance, a *net.Dialer -// with desired configuration). -func FromEnvironmentUsing(forward Dialer) Dialer { - allProxy := allProxyEnv.Get() - if len(allProxy) == 0 { - return forward - } - - proxyURL, err := url.Parse(allProxy) - if err != nil { - return forward - } - proxy, err := FromURL(proxyURL, forward) - if err != nil { - return forward - } - - noProxy := noProxyEnv.Get() - if len(noProxy) == 0 { - return proxy - } - - perHost := NewPerHost(proxy, forward) - perHost.AddFromString(noProxy) - return perHost -} - -// proxySchemes is a map from URL schemes to a function that creates a Dialer -// from a URL with such a scheme. -var proxySchemes map[string]func(*url.URL, Dialer) (Dialer, error) - -// RegisterDialerType takes a URL scheme and a function to generate Dialers from -// a URL with that scheme and a forwarding Dialer. Registered schemes are used -// by FromURL. -func RegisterDialerType(scheme string, f func(*url.URL, Dialer) (Dialer, error)) { - if proxySchemes == nil { - proxySchemes = make(map[string]func(*url.URL, Dialer) (Dialer, error)) - } - proxySchemes[scheme] = f -} - -// FromURL returns a Dialer given a URL specification and an underlying -// Dialer for it to make network requests. -func FromURL(u *url.URL, forward Dialer) (Dialer, error) { - var auth *Auth - if u.User != nil { - auth = new(Auth) - auth.User = u.User.Username() - if p, ok := u.User.Password(); ok { - auth.Password = p - } - } - - switch u.Scheme { - case "socks5", "socks5h": - addr := u.Hostname() - port := u.Port() - if port == "" { - port = "1080" - } - return SOCKS5("tcp", net.JoinHostPort(addr, port), auth, forward) - } - - // If the scheme doesn't match any of the built-in schemes, see if it - // was registered by another package. - if proxySchemes != nil { - if f, ok := proxySchemes[u.Scheme]; ok { - return f(u, forward) - } - } - - return nil, errors.New("proxy: unknown scheme: " + u.Scheme) -} - -var ( - allProxyEnv = &envOnce{ - names: []string{"ALL_PROXY", "all_proxy"}, - } - noProxyEnv = &envOnce{ - names: []string{"NO_PROXY", "no_proxy"}, - } -) - -// envOnce looks up an environment variable (optionally by multiple -// names) once. It mitigates expensive lookups on some platforms -// (e.g. Windows). -// (Borrowed from net/http/transport.go) -type envOnce struct { - names []string - once sync.Once - val string -} - -func (e *envOnce) Get() string { - e.once.Do(e.init) - return e.val -} - -func (e *envOnce) init() { - for _, n := range e.names { - e.val = os.Getenv(n) - if e.val != "" { - return - } - } -} - -// reset is used by tests -func (e *envOnce) reset() { - e.once = sync.Once{} - e.val = "" -} diff --git a/vendor/golang.org/x/net/proxy/socks5.go b/vendor/golang.org/x/net/proxy/socks5.go deleted file mode 100644 index c91651f9..00000000 --- a/vendor/golang.org/x/net/proxy/socks5.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package proxy - -import ( - "context" - "net" - - "golang.org/x/net/internal/socks" -) - -// SOCKS5 returns a Dialer that makes SOCKSv5 connections to the given -// address with an optional username and password. -// See RFC 1928 and RFC 1929. -func SOCKS5(network, address string, auth *Auth, forward Dialer) (Dialer, error) { - d := socks.NewDialer(network, address) - if forward != nil { - if f, ok := forward.(ContextDialer); ok { - d.ProxyDial = func(ctx context.Context, network string, address string) (net.Conn, error) { - return f.DialContext(ctx, network, address) - } - } else { - d.ProxyDial = func(ctx context.Context, network string, address string) (net.Conn, error) { - return dialContext(ctx, forward, network, address) - } - } - } - if auth != nil { - up := socks.UsernamePassword{ - Username: auth.User, - Password: auth.Password, - } - d.AuthMethods = []socks.AuthMethod{ - socks.AuthMethodNotRequired, - socks.AuthMethodUsernamePassword, - } - d.Authenticate = up.Authenticate - } - return d, nil -} diff --git a/vendor/modules.txt b/vendor/modules.txt index d3c3ed82..09c9bb26 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,38 +1,43 @@ -# github.com/aymerick/douceur v0.2.0 -## explicit -github.com/aymerick/douceur/css -github.com/aymerick/douceur/parser # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew +# github.com/didip/tollbooth/v7 v7.0.1 +## explicit; go 1.12 +github.com/didip/tollbooth/v7 +github.com/didip/tollbooth/v7/errors +github.com/didip/tollbooth/v7/internal/time/rate +github.com/didip/tollbooth/v7/libstring +github.com/didip/tollbooth/v7/limiter +# github.com/didip/tollbooth_chi v0.0.0-20220719025231-d662a7f6928f +## explicit; go 1.14 +github.com/didip/tollbooth_chi # github.com/fatih/color v1.16.0 ## explicit; go 1.17 github.com/fatih/color # github.com/fsnotify/fsnotify v1.7.0 ## explicit; go 1.17 github.com/fsnotify/fsnotify -# github.com/go-pkgz/email v0.4.1 -## explicit; go 1.19 -github.com/go-pkgz/email +# github.com/go-chi/chi v1.5.5 +## explicit; go 1.16 +github.com/go-chi/chi +# github.com/go-chi/chi/v5 v5.0.10 +## explicit; go 1.14 +github.com/go-chi/chi/v5 +github.com/go-chi/chi/v5/middleware +# github.com/go-pkgz/expirable-cache v0.1.0 +## explicit; go 1.14 +github.com/go-pkgz/expirable-cache # github.com/go-pkgz/lgr v0.11.1 ## explicit; go 1.20 github.com/go-pkgz/lgr -# github.com/go-pkgz/notify v1.0.0 -## explicit; go 1.20 -github.com/go-pkgz/notify -# github.com/go-pkgz/repeater v1.1.3 -## explicit; go 1.12 -github.com/go-pkgz/repeater -github.com/go-pkgz/repeater/strategy +# github.com/go-pkgz/rest v1.18.2 +## explicit; go 1.21 +github.com/go-pkgz/rest +github.com/go-pkgz/rest/logger +github.com/go-pkgz/rest/realip # github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1 ## explicit; go 1.16 github.com/go-telegram-bot-api/telegram-bot-api/v5 -# github.com/gorilla/css v1.0.1 -## explicit; go 1.20 -github.com/gorilla/css/scanner -# github.com/gorilla/websocket v1.5.1 -## explicit; go 1.20 -github.com/gorilla/websocket # github.com/hashicorp/errwrap v1.1.0 ## explicit github.com/hashicorp/errwrap @@ -45,20 +50,9 @@ github.com/mattn/go-colorable # github.com/mattn/go-isatty v0.0.20 ## explicit; go 1.15 github.com/mattn/go-isatty -# github.com/microcosm-cc/bluemonday v1.0.26 -## explicit; go 1.21 -github.com/microcosm-cc/bluemonday -github.com/microcosm-cc/bluemonday/css # github.com/pmezard/go-difflib v1.0.0 ## explicit github.com/pmezard/go-difflib/difflib -# github.com/slack-go/slack v0.12.3 -## explicit; go 1.16 -github.com/slack-go/slack -github.com/slack-go/slack/internal/backoff -github.com/slack-go/slack/internal/errorsx -github.com/slack-go/slack/internal/timex -github.com/slack-go/slack/slackutilsx # github.com/stretchr/testify v1.8.4 ## explicit; go 1.20 github.com/stretchr/testify/assert @@ -66,12 +60,6 @@ github.com/stretchr/testify/require # github.com/umputun/go-flags v1.5.1 ## explicit; go 1.12 github.com/umputun/go-flags -# golang.org/x/net v0.19.0 -## explicit; go 1.18 -golang.org/x/net/html -golang.org/x/net/html/atom -golang.org/x/net/internal/socks -golang.org/x/net/proxy # golang.org/x/sys v0.15.0 ## explicit; go 1.18 golang.org/x/sys/unix