aboutsummaryrefslogtreecommitdiffstats
path: root/walrss/internal
diff options
context:
space:
mode:
Diffstat (limited to 'walrss/internal')
-rw-r--r--walrss/internal/core/users.go25
-rw-r--r--walrss/internal/db/sendDay.go9
-rw-r--r--walrss/internal/http/views/main.qtpl.html14
-rw-r--r--walrss/internal/http/views/main.qtpl.html.go22
-rw-r--r--walrss/internal/rss/processor.go187
-rw-r--r--walrss/internal/rss/watcher.go26
-rw-r--r--walrss/internal/state/state.go7
7 files changed, 272 insertions, 18 deletions
diff --git a/walrss/internal/core/users.go b/walrss/internal/core/users.go
index e10da99..5071543 100644
--- a/walrss/internal/core/users.go
+++ b/walrss/internal/core/users.go
@@ -88,3 +88,28 @@ func UpdateUser(st *state.State, user *db.User) error {
}
return nil
}
+
+func GetUsersBySchedule(st *state.State, day db.SendDay, hour int) ([]*db.User, error) {
+ // When trying to Or some queries, BH was weird, so it's easier to make two queries and combine them.
+ // This ensures that indexes are used.
+
+ var users []*db.User
+ if err := st.Data.Find(&users,
+ bh.Where("Schedule.Active").Eq(true).
+ And("Schedule.Day").Eq(day).
+ And("Schedule.Hour").Eq(hour),
+ ); err != nil {
+ return nil, err
+ }
+
+ var users2 []*db.User
+ if err := st.Data.Find(&users2,
+ bh.Where("Schedule.Active").Eq(true).
+ And("Schedule.Day").Eq(db.SendDaily).
+ And("Schedule.Hour").Eq(hour),
+ ); err != nil {
+ return nil, err
+ }
+
+ return append(users, users2...), nil
+}
diff --git a/walrss/internal/db/sendDay.go b/walrss/internal/db/sendDay.go
index 5146a56..3f3d496 100644
--- a/walrss/internal/db/sendDay.go
+++ b/walrss/internal/db/sendDay.go
@@ -3,6 +3,7 @@ package db
import (
"errors"
"strings"
+ "time"
)
type SendDay uint32
@@ -78,3 +79,11 @@ func (s *SendDay) UnmarshalText(x []byte) error {
return nil
}
+
+func SendDayFromWeekday(w time.Weekday) SendDay {
+ s := new(SendDay)
+ if err := s.UnmarshalText([]byte(w.String())); err != nil {
+ panic(err)
+ }
+ return *s
+}
diff --git a/walrss/internal/http/views/main.qtpl.html b/walrss/internal/http/views/main.qtpl.html
index 34a92c6..fb5514b 100644
--- a/walrss/internal/http/views/main.qtpl.html
+++ b/walrss/internal/http/views/main.qtpl.html
@@ -68,7 +68,7 @@
<table class="table">
<thead>
- <tr>
+ <tr style="background: white; width: 100%; position: sticky; top: 0; border-bottom: black 1px solid;">
<th scope="col">Name</th>
<th scope="col">URL</th>
<th scope="col">
@@ -78,7 +78,7 @@
class="btn btn-primary"
hx-get="{%s= urls.NewFeedItem %}"
hx-target="#feedListing"
- hx-swap="beforeend"
+ hx-swap="beforeend show:bottom"
>
<i class="bi bi-plus"></i>
</button>
@@ -122,7 +122,7 @@
{% endfunc %}
{% func RenderFeedEditRow(id, name, url string) %}
-<tr id="feed-{%s= id %}" class="align-middle" hx-target="this" hx-swap="outerHTML">
+<tr id="feed-{%s= id %}" class="align-middle alert alert-warning" hx-target="this" hx-swap="outerHTML">
<th scope="row"><input
class="form-control form-control-sm"
type="text"
@@ -155,16 +155,16 @@
{% func RenderNewFeedItemRow() %}
{% code id := shortuuid.New() %}
-<tr id="{%s= id %}" class="align-middle" hx-target="this" hx-swap="outerHTML">
+<tr id="{%s= id %}" class="align-middle alert alert-warning" hx-target="this" hx-swap="outerHTML">
<th scope="row"><input
- id="{%s= id %}-name-input"
+ id="input-{%s= id %}-name"
class="form-control form-control-sm"
type="text"
name="name"
placeholder="Name"
></th>
<td><input
- id="{%s= id %}-url-input"
+ id="input-{%s= id %}-url"
class="form-control form-control-sm"
type="url"
name="url"
@@ -176,7 +176,7 @@
type="button"
class="btn btn-outline-success"
hx-post="{%s= urls.NewFeedItem %}"
- hx-include="#{%s= id %}-name-input, #{%s= id %}-url-input">
+ hx-include="#input-{%s= id %}-name, #input-{%s= id %}-url">
<i class="bi bi-check"></i>
</button>
<button type="button" class="btn btn-outline-danger" id="{%s= id %}-cancel"><i class="bi bi-x"></i></button>
diff --git a/walrss/internal/http/views/main.qtpl.html.go b/walrss/internal/http/views/main.qtpl.html.go
index 55be033..4eb29c4 100644
--- a/walrss/internal/http/views/main.qtpl.html.go
+++ b/walrss/internal/http/views/main.qtpl.html.go
@@ -107,7 +107,7 @@ func (p *MainPage) StreamBody(qw422016 *qt422016.Writer) {
<table class="table">
<thead>
- <tr>
+ <tr style="background: white; width: 100%; position: sticky; top: 0; border-bottom: black 1px solid;">
<th scope="col">Name</th>
<th scope="col">URL</th>
<th scope="col">
@@ -119,7 +119,7 @@ func (p *MainPage) StreamBody(qw422016 *qt422016.Writer) {
qw422016.N().S(urls.NewFeedItem)
qw422016.N().S(`"
hx-target="#feedListing"
- hx-swap="beforeend"
+ hx-swap="beforeend show:bottom"
>
<i class="bi bi-plus"></i>
</button>
@@ -217,7 +217,7 @@ func StreamRenderFeedEditRow(qw422016 *qt422016.Writer, id, name, url string) {
qw422016.N().S(`
<tr id="feed-`)
qw422016.N().S(id)
- qw422016.N().S(`" class="align-middle" hx-target="this" hx-swap="outerHTML">
+ qw422016.N().S(`" class="align-middle alert alert-warning" hx-target="this" hx-swap="outerHTML">
<th scope="row"><input
class="form-control form-control-sm"
type="text"
@@ -287,20 +287,20 @@ func StreamRenderNewFeedItemRow(qw422016 *qt422016.Writer) {
qw422016.N().S(`
<tr id="`)
qw422016.N().S(id)
- qw422016.N().S(`" class="align-middle" hx-target="this" hx-swap="outerHTML">
+ qw422016.N().S(`" class="align-middle alert alert-warning" hx-target="this" hx-swap="outerHTML">
<th scope="row"><input
- id="`)
+ id="input-`)
qw422016.N().S(id)
- qw422016.N().S(`-name-input"
+ qw422016.N().S(`-name"
class="form-control form-control-sm"
type="text"
name="name"
placeholder="Name"
></th>
<td><input
- id="`)
+ id="input-`)
qw422016.N().S(id)
- qw422016.N().S(`-url-input"
+ qw422016.N().S(`-url"
class="form-control form-control-sm"
type="url"
name="url"
@@ -314,11 +314,11 @@ func StreamRenderNewFeedItemRow(qw422016 *qt422016.Writer) {
hx-post="`)
qw422016.N().S(urls.NewFeedItem)
qw422016.N().S(`"
- hx-include="#`)
+ hx-include="#input-`)
qw422016.N().S(id)
- qw422016.N().S(`-name-input, #`)
+ qw422016.N().S(`-name, #input-`)
qw422016.N().S(id)
- qw422016.N().S(`-url-input">
+ qw422016.N().S(`-url">
<i class="bi bi-check"></i>
</button>
<button type="button" class="btn btn-outline-danger" id="`)
diff --git a/walrss/internal/rss/processor.go b/walrss/internal/rss/processor.go
new file mode 100644
index 0000000..e220f0a
--- /dev/null
+++ b/walrss/internal/rss/processor.go
@@ -0,0 +1,187 @@
+package rss
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "github.com/carlmjohnson/requests"
+ "github.com/codemicro/walrss/walrss/internal/core"
+ "github.com/codemicro/walrss/walrss/internal/db"
+ "github.com/codemicro/walrss/walrss/internal/state"
+ "github.com/matcornic/hermes"
+ "github.com/mmcdole/gofeed"
+ "github.com/patrickmn/go-cache"
+ "io/ioutil"
+ "sort"
+ "strings"
+ "sync"
+ "time"
+)
+
+const (
+ dateFormat = "02Jan06"
+ timeFormat = "15:04:05"
+)
+
+type processedFeed struct {
+ Name string
+ Items []*feedItem
+ Error error
+}
+
+func ProcessFeeds(st *state.State, day db.SendDay, hour int) error {
+ u, e := core.GetUsersBySchedule(st, day, hour)
+ for _, ur := range u {
+ fmt.Printf("%#v\n", ur)
+
+ userFeeds, err := core.GetFeedsForUser(st, ur.ID)
+ if err != nil {
+ return err
+ }
+
+ var processedFeeds []*processedFeed
+
+ for _, f := range userFeeds {
+ pf := new(processedFeed)
+ pf.Name = f.Name
+
+ rawFeed, err := getFeedContent(f.URL)
+ if err != nil {
+ pf.Error = err
+ } else {
+ pf.Items = filterFeedContent(rawFeed, time.Date(2022, 04, 01, 0, 0, 0, 0, time.UTC))
+ }
+ processedFeeds = append(processedFeeds, pf)
+ }
+
+ plainContent, htmlContent, err := generateEmail(processedFeeds)
+ if err != nil {
+ return err
+ }
+
+ // TODO: Send email
+ }
+
+ return nil
+}
+
+var (
+ feedCache = cache.New(time.Minute*10, time.Minute*20)
+ feedFetchLock = new(sync.Mutex)
+)
+
+func getFeedContent(url string) (*gofeed.Feed, error) {
+ feedFetchLock.Lock()
+ defer feedFetchLock.Unlock()
+
+ if v, found := feedCache.Get(url); found {
+ return v.(*gofeed.Feed), nil
+ }
+
+ buf := new(bytes.Buffer)
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+ defer cancel()
+
+ if err := requests.URL(url).ToBytesBuffer(buf).Fetch(ctx); err != nil {
+ return nil, err
+ }
+
+ feed, err := gofeed.NewParser().Parse(buf)
+ if err != nil {
+ return nil, err
+ }
+
+ _ = feedCache.Add(url, feed, cache.DefaultExpiration)
+
+ return feed, nil
+}
+
+type feedItem struct {
+ Title string
+ URL string
+ PublishTime time.Time
+}
+
+func filterFeedContent(feed *gofeed.Feed, earliestPublishTime time.Time) []*feedItem {
+ var o []*feedItem
+
+ for _, item := range feed.Items {
+ if item.PublishedParsed != nil && item.PublishedParsed.After(earliestPublishTime) {
+ o = append(o, &feedItem{
+ Title: item.Title,
+ URL: item.Link,
+ PublishTime: *item.PublishedParsed,
+ })
+ }
+ }
+
+ return o
+}
+
+func generateEmail(processedItems []*processedFeed) (plain, html []byte, err error) {
+ sort.Slice(processedItems, func(i, j int) bool {
+ pi, pj := processedItems[i], processedItems[j]
+
+ if pi.Error != nil && pj.Error == nil {
+ return false
+ }
+
+ if pi.Error == nil && pj.Error != nil {
+ return true
+ }
+
+ return pi.Name < pj.Name
+ })
+
+ var sb strings.Builder
+
+ for _, processedItem := range processedItems {
+
+ if len(processedItem.Items) != 0 || processedItem.Error != nil {
+ sb.WriteString("* **")
+ sb.WriteString(strings.ReplaceAll(processedItem.Name, "*", `\*`))
+ sb.WriteString("**\n")
+ }
+
+ if processedItem.Error != nil {
+ sb.WriteString(" * **Error:** ")
+ sb.WriteString(processedItem.Error.Error())
+ sb.WriteString("\n")
+ } else {
+ r := strings.NewReplacer("[", `\[`, "]", `\]`, "*", `\*`)
+
+ for _, item := range processedItem.Items {
+ sb.WriteString(" * [**")
+ sb.WriteString(r.Replace(item.Title))
+ sb.WriteString("**](")
+ sb.WriteString(item.URL)
+ sb.WriteString(") - ")
+ sb.WriteString(strings.ToUpper(item.PublishTime.Format(dateFormat + " " + timeFormat)))
+ sb.WriteString("\n")
+ }
+
+ }
+ }
+
+ e := hermes.Email{
+ Body: hermes.Body{
+ FreeMarkdown: hermes.Markdown(sb.String()),
+ },
+ }
+
+ renderer := hermes.Hermes{
+ Theme: new(hermes.Flat),
+ }
+
+ plainString, err := renderer.GeneratePlainText(e)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ htmlString, err := renderer.GenerateHTML(e)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ return []byte(plainString), []byte(htmlString), nil
+}
diff --git a/walrss/internal/rss/watcher.go b/walrss/internal/rss/watcher.go
new file mode 100644
index 0000000..e7036c1
--- /dev/null
+++ b/walrss/internal/rss/watcher.go
@@ -0,0 +1,26 @@
+package rss
+
+import (
+ "github.com/codemicro/walrss/walrss/internal/db"
+ "github.com/codemicro/walrss/walrss/internal/state"
+ "github.com/rs/zerolog/log"
+ "time"
+)
+
+func StartWatcher(st *state.State) {
+ go func() {
+ currentTime := time.Now().UTC()
+ time.Sleep(time.Minute * time.Duration(60-currentTime.Minute()))
+
+ if err := ProcessFeeds(st, db.SendDayFromWeekday(currentTime.Weekday()), currentTime.Hour()+1); err != nil {
+ log.Error().Err(err).Str("location", "feed watcher").Send()
+ }
+
+ ticker := time.NewTicker(time.Hour)
+ for currentTime := range ticker.C {
+ if err := ProcessFeeds(st, db.SendDayFromWeekday(currentTime.Weekday()), currentTime.Hour()); err != nil {
+ log.Error().Err(err).Str("location", "feed watcher").Send()
+ }
+ }
+ }()
+}
diff --git a/walrss/internal/state/state.go b/walrss/internal/state/state.go
index 54badd7..31cb59b 100644
--- a/walrss/internal/state/state.go
+++ b/walrss/internal/state/state.go
@@ -19,6 +19,13 @@ func New() *State {
}
type Config struct {
+ //Email struct {
+ // Host string `fig:"host" validate:"required"`
+ // Username string `fig:"username" validate:"required"`
+ // Password string `fig:"password" validate:"required"`
+ // From string `fig:"from" validate:"required"`
+ // Port int `fig:"port" validate:"required"`
+ //}
Server struct {
Host string `fig:"host" default:"127.0.0.1"`
Port int `fig:"port" default:"8080"`
generated by cgit v1.2.3-70-g09d2 (git 2.51.2) at 2025-11-25 11:46:23 +0000