diff options
| -rw-r--r-- | go.mod | 27 | ||||
| -rw-r--r-- | go.sum | 64 | ||||
| -rw-r--r-- | walrss/internal/core/users.go | 25 | ||||
| -rw-r--r-- | walrss/internal/db/sendDay.go | 9 | ||||
| -rw-r--r-- | walrss/internal/http/views/main.qtpl.html | 14 | ||||
| -rw-r--r-- | walrss/internal/http/views/main.qtpl.html.go | 22 | ||||
| -rw-r--r-- | walrss/internal/rss/processor.go | 187 | ||||
| -rw-r--r-- | walrss/internal/rss/watcher.go | 26 | ||||
| -rw-r--r-- | walrss/internal/state/state.go | 7 | ||||
| -rw-r--r-- | walrss/main.go | 3 |
10 files changed, 365 insertions, 19 deletions
@@ -3,16 +3,38 @@ module github.com/codemicro/walrss go 1.18 require ( + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver v1.5.0 // indirect + github.com/Masterminds/sprig v2.22.0+incompatible // indirect + github.com/PuerkitoBio/goquery v1.5.1 // indirect github.com/andybalholm/brotli v1.0.4 // indirect + github.com/andybalholm/cascadia v1.1.0 // indirect github.com/bwmarrin/go-alone v0.0.0-20190806015146-742bb55d1631 // indirect + github.com/carlmjohnson/requests v0.22.2 // indirect github.com/gofiber/fiber/v2 v2.31.0 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/huandu/xstrings v1.3.2 // indirect + github.com/imdario/mergo v0.3.12 // indirect + github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba // indirect + github.com/json-iterator/go v1.1.10 // indirect github.com/kkyr/fig v0.3.0 // indirect github.com/klauspost/compress v1.15.0 // indirect github.com/lithammer/shortuuid/v4 v4.0.0 // indirect + github.com/matcornic/hermes v1.2.0 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/mmcdole/gofeed v1.1.3 // indirect + github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf // indirect + github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect + github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pelletier/go-toml v1.9.3 // indirect github.com/rs/zerolog v1.26.1 // indirect + github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect + github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect github.com/stevelacy/daz v0.1.4 // indirect github.com/timshannon/bolthold v0.0.0-20210913165410-232392fc8a6a // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect @@ -21,6 +43,9 @@ require ( github.com/valyala/tcplisten v1.0.0 // indirect go.etcd.io/bbolt v1.3.6 // indirect golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect + golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect + golang.org/x/text v0.3.7 // indirect + gopkg.in/russross/blackfriday.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect -)
\ No newline at end of file +) @@ -1,16 +1,41 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60= +github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= +github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/bwmarrin/go-alone v0.0.0-20190806015146-742bb55d1631 h1:Xb5rra6jJt5Z1JsZhIMby+IP5T8aU+Uc2RC9RzSxs9g= github.com/bwmarrin/go-alone v0.0.0-20190806015146-742bb55d1631/go.mod h1:P86Dksd9km5HGX5UMIocXvX87sEp2xUARle3by+9JZ4= +github.com/carlmjohnson/requests v0.22.2 h1:hccG5g9ITJlnDip54OVa810AkB366kthFjvA90N4owM= +github.com/carlmjohnson/requests v0.22.2/go.mod h1:Hw4fFOk3xDlHQbNRTGo4oc52TUTpVEq93sNy/H+mrQM= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofiber/fiber/v2 v2.31.0 h1:M2rWPQbD5fDVAjcoOLjKRXTIlHesI5Eq7I5FEQPt4Ow= github.com/gofiber/fiber/v2 v2.31.0/go.mod h1:1Ega6O199a3Y7yDGuM9FyXDPYQfv+7/y48wl6WCwUF4= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= +github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba h1:QFQpJdgbON7I0jr2hYW7Bs+XV0qjc3d5tZoDnRFnqTg= +github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/kkyr/fig v0.3.0 h1:5bd1amYKp/gsK2bGEUJYzcCrQPKOZp6HZD9K21v9Guo= github.com/kkyr/fig v0.3.0/go.mod h1:fEnrLjwg/iwSr8ksJF4DxrDmCUir5CaVMLORGYMcz30= github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= @@ -19,18 +44,47 @@ github.com/klauspost/compress v1.15.0 h1:xqfchp4whNFxn5A4XFyyYtitiWI8Hy5EW59jEwc github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw7k08o4c= github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= +github.com/matcornic/hermes v1.2.0 h1:AuqZpYcTOtTB7cahdevLfnhIpfzmpqw5Czv8vpdnFDU= +github.com/matcornic/hermes v1.2.0/go.mod h1:lujJomb016Xjv8wBnWlNvUdtmvowjjfkqri5J/+1hYc= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o= +github.com/mmcdole/gofeed v1.1.3/go.mod h1:QQO3maftbOu+hiVOGOZDRLymqGQCos4zxbA4j89gMrE= +github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI= +github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo= +github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM= github.com/stevelacy/daz v0.1.4 h1:ugmff/D7D764wZjXSgSryEINE/bi+Xddllw3JQQGbWk= github.com/stevelacy/daz v0.1.4/go.mod h1:AbK6DzjiIL15r4bQtcFvOBAvDGMXoh+uIG26NRUugt0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/timshannon/bolthold v0.0.0-20210913165410-232392fc8a6a h1:oIi7H/bwFUYKYhzKbHc+3MvHRWqhQwXVB4LweLMiVy0= github.com/timshannon/bolthold v0.0.0-20210913165410-232392fc8a6a/go.mod h1:iSvujNDmpZ6eQX+bg/0X3lF7LEmZ8N77g2a/J/+Zt2U= +github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.30.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus= @@ -50,12 +104,16 @@ golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -73,8 +131,10 @@ golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -83,5 +143,9 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/russross/blackfriday.v2 v2.0.0 h1:+FlnIV8DSQnT7NZ43hcVKcdJdzZoeCmJj4Ql8gq5keA= +gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/walrss/internal/core/users.go b/walrss/internal/core/users.go index e10da99..5071543 100644 --- a/walrss/internal/core/users.go +++ b/walrss/internal/core/users.go @@ -88,3 +88,28 @@ func UpdateUser(st *state.State, user *db.User) error { } return nil } + +func GetUsersBySchedule(st *state.State, day db.SendDay, hour int) ([]*db.User, error) { + // When trying to Or some queries, BH was weird, so it's easier to make two queries and combine them. + // This ensures that indexes are used. + + var users []*db.User + if err := st.Data.Find(&users, + bh.Where("Schedule.Active").Eq(true). + And("Schedule.Day").Eq(day). + And("Schedule.Hour").Eq(hour), + ); err != nil { + return nil, err + } + + var users2 []*db.User + if err := st.Data.Find(&users2, + bh.Where("Schedule.Active").Eq(true). + And("Schedule.Day").Eq(db.SendDaily). + And("Schedule.Hour").Eq(hour), + ); err != nil { + return nil, err + } + + return append(users, users2...), nil +} diff --git a/walrss/internal/db/sendDay.go b/walrss/internal/db/sendDay.go index 5146a56..3f3d496 100644 --- a/walrss/internal/db/sendDay.go +++ b/walrss/internal/db/sendDay.go @@ -3,6 +3,7 @@ package db import ( "errors" "strings" + "time" ) type SendDay uint32 @@ -78,3 +79,11 @@ func (s *SendDay) UnmarshalText(x []byte) error { return nil } + +func SendDayFromWeekday(w time.Weekday) SendDay { + s := new(SendDay) + if err := s.UnmarshalText([]byte(w.String())); err != nil { + panic(err) + } + return *s +} diff --git a/walrss/internal/http/views/main.qtpl.html b/walrss/internal/http/views/main.qtpl.html index 34a92c6..fb5514b 100644 --- a/walrss/internal/http/views/main.qtpl.html +++ b/walrss/internal/http/views/main.qtpl.html @@ -68,7 +68,7 @@ <table class="table"> <thead> - <tr> + <tr style="background: white; width: 100%; position: sticky; top: 0; border-bottom: black 1px solid;"> <th scope="col">Name</th> <th scope="col">URL</th> <th scope="col"> @@ -78,7 +78,7 @@ class="btn btn-primary" hx-get="{%s= urls.NewFeedItem %}" hx-target="#feedListing" - hx-swap="beforeend" + hx-swap="beforeend show:bottom" > <i class="bi bi-plus"></i> </button> @@ -122,7 +122,7 @@ {% endfunc %} {% func RenderFeedEditRow(id, name, url string) %} -<tr id="feed-{%s= id %}" class="align-middle" hx-target="this" hx-swap="outerHTML"> +<tr id="feed-{%s= id %}" class="align-middle alert alert-warning" hx-target="this" hx-swap="outerHTML"> <th scope="row"><input class="form-control form-control-sm" type="text" @@ -155,16 +155,16 @@ {% func RenderNewFeedItemRow() %} {% code id := shortuuid.New() %} -<tr id="{%s= id %}" class="align-middle" hx-target="this" hx-swap="outerHTML"> +<tr id="{%s= id %}" class="align-middle alert alert-warning" hx-target="this" hx-swap="outerHTML"> <th scope="row"><input - id="{%s= id %}-name-input" + id="input-{%s= id %}-name" class="form-control form-control-sm" type="text" name="name" placeholder="Name" ></th> <td><input - id="{%s= id %}-url-input" + id="input-{%s= id %}-url" class="form-control form-control-sm" type="url" name="url" @@ -176,7 +176,7 @@ type="button" class="btn btn-outline-success" hx-post="{%s= urls.NewFeedItem %}" - hx-include="#{%s= id %}-name-input, #{%s= id %}-url-input"> + hx-include="#input-{%s= id %}-name, #input-{%s= id %}-url"> <i class="bi bi-check"></i> </button> <button type="button" class="btn btn-outline-danger" id="{%s= id %}-cancel"><i class="bi bi-x"></i></button> diff --git a/walrss/internal/http/views/main.qtpl.html.go b/walrss/internal/http/views/main.qtpl.html.go index 55be033..4eb29c4 100644 --- a/walrss/internal/http/views/main.qtpl.html.go +++ b/walrss/internal/http/views/main.qtpl.html.go @@ -107,7 +107,7 @@ func (p *MainPage) StreamBody(qw422016 *qt422016.Writer) { <table class="table"> <thead> - <tr> + <tr style="background: white; width: 100%; position: sticky; top: 0; border-bottom: black 1px solid;"> <th scope="col">Name</th> <th scope="col">URL</th> <th scope="col"> @@ -119,7 +119,7 @@ func (p *MainPage) StreamBody(qw422016 *qt422016.Writer) { qw422016.N().S(urls.NewFeedItem) qw422016.N().S(`" hx-target="#feedListing" - hx-swap="beforeend" + hx-swap="beforeend show:bottom" > <i class="bi bi-plus"></i> </button> @@ -217,7 +217,7 @@ func StreamRenderFeedEditRow(qw422016 *qt422016.Writer, id, name, url string) { qw422016.N().S(` <tr id="feed-`) qw422016.N().S(id) - qw422016.N().S(`" class="align-middle" hx-target="this" hx-swap="outerHTML"> + qw422016.N().S(`" class="align-middle alert alert-warning" hx-target="this" hx-swap="outerHTML"> <th scope="row"><input class="form-control form-control-sm" type="text" @@ -287,20 +287,20 @@ func StreamRenderNewFeedItemRow(qw422016 *qt422016.Writer) { qw422016.N().S(` <tr id="`) qw422016.N().S(id) - qw422016.N().S(`" class="align-middle" hx-target="this" hx-swap="outerHTML"> + qw422016.N().S(`" class="align-middle alert alert-warning" hx-target="this" hx-swap="outerHTML"> <th scope="row"><input - id="`) + id="input-`) qw422016.N().S(id) - qw422016.N().S(`-name-input" + qw422016.N().S(`-name" class="form-control form-control-sm" type="text" name="name" placeholder="Name" ></th> <td><input - id="`) + id="input-`) qw422016.N().S(id) - qw422016.N().S(`-url-input" + qw422016.N().S(`-url" class="form-control form-control-sm" type="url" name="url" @@ -314,11 +314,11 @@ func StreamRenderNewFeedItemRow(qw422016 *qt422016.Writer) { hx-post="`) qw422016.N().S(urls.NewFeedItem) qw422016.N().S(`" - hx-include="#`) + hx-include="#input-`) qw422016.N().S(id) - qw422016.N().S(`-name-input, #`) + qw422016.N().S(`-name, #input-`) qw422016.N().S(id) - qw422016.N().S(`-url-input"> + qw422016.N().S(`-url"> <i class="bi bi-check"></i> </button> <button type="button" class="btn btn-outline-danger" id="`) diff --git a/walrss/internal/rss/processor.go b/walrss/internal/rss/processor.go new file mode 100644 index 0000000..e220f0a --- /dev/null +++ b/walrss/internal/rss/processor.go @@ -0,0 +1,187 @@ +package rss + +import ( + "bytes" + "context" + "fmt" + "github.com/carlmjohnson/requests" + "github.com/codemicro/walrss/walrss/internal/core" + "github.com/codemicro/walrss/walrss/internal/db" + "github.com/codemicro/walrss/walrss/internal/state" + "github.com/matcornic/hermes" + "github.com/mmcdole/gofeed" + "github.com/patrickmn/go-cache" + "io/ioutil" + "sort" + "strings" + "sync" + "time" +) + +const ( + dateFormat = "02Jan06" + timeFormat = "15:04:05" +) + +type processedFeed struct { + Name string + Items []*feedItem + Error error +} + +func ProcessFeeds(st *state.State, day db.SendDay, hour int) error { + u, e := core.GetUsersBySchedule(st, day, hour) + for _, ur := range u { + fmt.Printf("%#v\n", ur) + + userFeeds, err := core.GetFeedsForUser(st, ur.ID) + if err != nil { + return err + } + + var processedFeeds []*processedFeed + + for _, f := range userFeeds { + pf := new(processedFeed) + pf.Name = f.Name + + rawFeed, err := getFeedContent(f.URL) + if err != nil { + pf.Error = err + } else { + pf.Items = filterFeedContent(rawFeed, time.Date(2022, 04, 01, 0, 0, 0, 0, time.UTC)) + } + processedFeeds = append(processedFeeds, pf) + } + + plainContent, htmlContent, err := generateEmail(processedFeeds) + if err != nil { + return err + } + + // TODO: Send email + } + + return nil +} + +var ( + feedCache = cache.New(time.Minute*10, time.Minute*20) + feedFetchLock = new(sync.Mutex) +) + +func getFeedContent(url string) (*gofeed.Feed, error) { + feedFetchLock.Lock() + defer feedFetchLock.Unlock() + + if v, found := feedCache.Get(url); found { + return v.(*gofeed.Feed), nil + } + + buf := new(bytes.Buffer) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + if err := requests.URL(url).ToBytesBuffer(buf).Fetch(ctx); err != nil { + return nil, err + } + + feed, err := gofeed.NewParser().Parse(buf) + if err != nil { + return nil, err + } + + _ = feedCache.Add(url, feed, cache.DefaultExpiration) + + return feed, nil +} + +type feedItem struct { + Title string + URL string + PublishTime time.Time +} + +func filterFeedContent(feed *gofeed.Feed, earliestPublishTime time.Time) []*feedItem { + var o []*feedItem + + for _, item := range feed.Items { + if item.PublishedParsed != nil && item.PublishedParsed.After(earliestPublishTime) { + o = append(o, &feedItem{ + Title: item.Title, + URL: item.Link, + PublishTime: *item.PublishedParsed, + }) + } + } + + return o +} + +func generateEmail(processedItems []*processedFeed) (plain, html []byte, err error) { + sort.Slice(processedItems, func(i, j int) bool { + pi, pj := processedItems[i], processedItems[j] + + if pi.Error != nil && pj.Error == nil { + return false + } + + if pi.Error == nil && pj.Error != nil { + return true + } + + return pi.Name < pj.Name + }) + + var sb strings.Builder + + for _, processedItem := range processedItems { + + if len(processedItem.Items) != 0 || processedItem.Error != nil { + sb.WriteString("* **") + sb.WriteString(strings.ReplaceAll(processedItem.Name, "*", `\*`)) + sb.WriteString("**\n") + } + + if processedItem.Error != nil { + sb.WriteString(" * **Error:** ") + sb.WriteString(processedItem.Error.Error()) + sb.WriteString("\n") + } else { + r := strings.NewReplacer("[", `\[`, "]", `\]`, "*", `\*`) + + for _, item := range processedItem.Items { + sb.WriteString(" * [**") + sb.WriteString(r.Replace(item.Title)) + sb.WriteString("**](") + sb.WriteString(item.URL) + sb.WriteString(") - ") + sb.WriteString(strings.ToUpper(item.PublishTime.Format(dateFormat + " " + timeFormat))) + sb.WriteString("\n") + } + + } + } + + e := hermes.Email{ + Body: hermes.Body{ + FreeMarkdown: hermes.Markdown(sb.String()), + }, + } + + renderer := hermes.Hermes{ + Theme: new(hermes.Flat), + } + + plainString, err := renderer.GeneratePlainText(e) + if err != nil { + return nil, nil, err + } + + htmlString, err := renderer.GenerateHTML(e) + if err != nil { + return nil, nil, err + } + + return []byte(plainString), []byte(htmlString), nil +} diff --git a/walrss/internal/rss/watcher.go b/walrss/internal/rss/watcher.go new file mode 100644 index 0000000..e7036c1 --- /dev/null +++ b/walrss/internal/rss/watcher.go @@ -0,0 +1,26 @@ +package rss + +import ( + "github.com/codemicro/walrss/walrss/internal/db" + "github.com/codemicro/walrss/walrss/internal/state" + "github.com/rs/zerolog/log" + "time" +) + +func StartWatcher(st *state.State) { + go func() { + currentTime := time.Now().UTC() + time.Sleep(time.Minute * time.Duration(60-currentTime.Minute())) + + if err := ProcessFeeds(st, db.SendDayFromWeekday(currentTime.Weekday()), currentTime.Hour()+1); err != nil { + log.Error().Err(err).Str("location", "feed watcher").Send() + } + + ticker := time.NewTicker(time.Hour) + for currentTime := range ticker.C { + if err := ProcessFeeds(st, db.SendDayFromWeekday(currentTime.Weekday()), currentTime.Hour()); err != nil { + log.Error().Err(err).Str("location", "feed watcher").Send() + } + } + }() +} diff --git a/walrss/internal/state/state.go b/walrss/internal/state/state.go index 54badd7..31cb59b 100644 --- a/walrss/internal/state/state.go +++ b/walrss/internal/state/state.go @@ -19,6 +19,13 @@ func New() *State { } type Config struct { + //Email struct { + // Host string `fig:"host" validate:"required"` + // Username string `fig:"username" validate:"required"` + // Password string `fig:"password" validate:"required"` + // From string `fig:"from" validate:"required"` + // Port int `fig:"port" validate:"required"` + //} Server struct { Host string `fig:"host" default:"127.0.0.1"` Port int `fig:"port" default:"8080"` diff --git a/walrss/main.go b/walrss/main.go index 9cb951d..9978bab 100644 --- a/walrss/main.go +++ b/walrss/main.go @@ -3,6 +3,7 @@ package main import ( "github.com/codemicro/walrss/walrss/internal/db" "github.com/codemicro/walrss/walrss/internal/http" + "github.com/codemicro/walrss/walrss/internal/rss" "github.com/codemicro/walrss/walrss/internal/state" "github.com/rs/zerolog/log" "os" @@ -38,6 +39,8 @@ func run() error { return err } + rss.ProcessFeeds(st, db.SendOnSunday, 21) + return server.Run() } |