Implement beginnings of email sending

Signed-off-by: AKP <tom@tdpain.net>

8 files changed, 275 insertions, 18 deletions

diff --git a/walrss/internal/core/users.go b/walrss/internal/core/users.go
index e10da99..5071543 100644
--- a/walrss/internal/core/users.go
+++ b/walrss/internal/core/users.go
@@ -88,3 +88,28 @@ func UpdateUser(st *state.State, user *db.User) error {
 	}
 	return nil
 }
+
+func GetUsersBySchedule(st *state.State, day db.SendDay, hour int) ([]*db.User, error) {
+	// When trying to Or some queries, BH was weird, so it's easier to make two queries and combine them.
+	// This ensures that indexes are used.
+
+	var users []*db.User
+	if err := st.Data.Find(&users,
+		bh.Where("Schedule.Active").Eq(true).
+			And("Schedule.Day").Eq(day).
+			And("Schedule.Hour").Eq(hour),
+	); err != nil {
+		return nil, err
+	}
+
+	var users2 []*db.User
+	if err := st.Data.Find(&users2,
+		bh.Where("Schedule.Active").Eq(true).
+			And("Schedule.Day").Eq(db.SendDaily).
+			And("Schedule.Hour").Eq(hour),
+	); err != nil {
+		return nil, err
+	}
+
+	return append(users, users2...), nil
+}
diff --git a/walrss/internal/db/sendDay.go b/walrss/internal/db/sendDay.go
index 5146a56..3f3d496 100644
--- a/walrss/internal/db/sendDay.go
+++ b/walrss/internal/db/sendDay.go
@@ -3,6 +3,7 @@ package db
 import (
 	"errors"
 	"strings"
+	"time"
 )
 
 type SendDay uint32
@@ -78,3 +79,11 @@ func (s *SendDay) UnmarshalText(x []byte) error {
 
 	return nil
 }
+
+func SendDayFromWeekday(w time.Weekday) SendDay {
+	s := new(SendDay)
+	if err := s.UnmarshalText([]byte(w.String())); err != nil {
+		panic(err)
+	}
+	return *s
+}
diff --git a/walrss/internal/http/views/main.qtpl.html b/walrss/internal/http/views/main.qtpl.html
index 34a92c6..fb5514b 100644
--- a/walrss/internal/http/views/main.qtpl.html
+++ b/walrss/internal/http/views/main.qtpl.html
@@ -68,7 +68,7 @@
 
             <table class="table">
                 <thead>
-                    <tr>
+                    <tr style="background: white; width: 100%; position: sticky; top: 0; border-bottom: black 1px solid;">
                         <th scope="col">Name</th>
                         <th scope="col">URL</th>
                         <th scope="col">
@@ -78,7 +78,7 @@
                                         class="btn btn-primary"
                                         hx-get="{%s= urls.NewFeedItem %}"
                                         hx-target="#feedListing"
-                                        hx-swap="beforeend"
+                                        hx-swap="beforeend show:bottom"
                                 >
                                     <i class="bi bi-plus"></i>
                                 </button>
@@ -122,7 +122,7 @@
 {% endfunc %}
 
 {% func RenderFeedEditRow(id, name, url string) %}
-<tr id="feed-{%s= id %}" class="align-middle" hx-target="this" hx-swap="outerHTML">
+<tr id="feed-{%s= id %}" class="align-middle alert alert-warning" hx-target="this" hx-swap="outerHTML">
     <th scope="row"><input
             class="form-control form-control-sm"
             type="text"
@@ -155,16 +155,16 @@
 
 {% func RenderNewFeedItemRow() %}
 {% code id := shortuuid.New() %}
-<tr id="{%s= id %}" class="align-middle" hx-target="this" hx-swap="outerHTML">
+<tr id="{%s= id %}" class="align-middle alert alert-warning" hx-target="this" hx-swap="outerHTML">
     <th scope="row"><input
-            id="{%s= id %}-name-input"
+            id="input-{%s= id %}-name"
             class="form-control form-control-sm"
             type="text"
             name="name"
             placeholder="Name"
     ></th>
     <td><input
-            id="{%s= id %}-url-input"
+            id="input-{%s= id %}-url"
             class="form-control form-control-sm"
             type="url"
             name="url"
@@ -176,7 +176,7 @@
                     type="button"
                     class="btn btn-outline-success"
                     hx-post="{%s= urls.NewFeedItem %}"
-                    hx-include="#{%s= id %}-name-input, #{%s= id %}-url-input">
+                    hx-include="#input-{%s= id %}-name, #input-{%s= id %}-url">
                 <i class="bi bi-check"></i>
             </button>
             <button type="button" class="btn btn-outline-danger" id="{%s= id %}-cancel"><i class="bi bi-x"></i></button>
diff --git a/walrss/internal/http/views/main.qtpl.html.go b/walrss/internal/http/views/main.qtpl.html.go
index 55be033..4eb29c4 100644
--- a/walrss/internal/http/views/main.qtpl.html.go
+++ b/walrss/internal/http/views/main.qtpl.html.go
@@ -107,7 +107,7 @@ func (p *MainPage) StreamBody(qw422016 *qt422016.Writer) {
 
             <table class="table">
                 <thead>
-                    <tr>
+                    <tr style="background: white; width: 100%; position: sticky; top: 0; border-bottom: black 1px solid;">
                         <th scope="col">Name</th>
                         <th scope="col">URL</th>
                         <th scope="col">
@@ -119,7 +119,7 @@ func (p *MainPage) StreamBody(qw422016 *qt422016.Writer) {
 	qw422016.N().S(urls.NewFeedItem)
 	qw422016.N().S(`"
                                         hx-target="#feedListing"
-                                        hx-swap="beforeend"
+                                        hx-swap="beforeend show:bottom"
                                 >
                                     <i class="bi bi-plus"></i>
                                 </button>
@@ -217,7 +217,7 @@ func StreamRenderFeedEditRow(qw422016 *qt422016.Writer, id, name, url string) {
 	qw422016.N().S(`
 <tr id="feed-`)
 	qw422016.N().S(id)
-	qw422016.N().S(`" class="align-middle" hx-target="this" hx-swap="outerHTML">
+	qw422016.N().S(`" class="align-middle alert alert-warning" hx-target="this" hx-swap="outerHTML">
     <th scope="row"><input
             class="form-control form-control-sm"
             type="text"
@@ -287,20 +287,20 @@ func StreamRenderNewFeedItemRow(qw422016 *qt422016.Writer) {
 	qw422016.N().S(`
 <tr id="`)
 	qw422016.N().S(id)
-	qw422016.N().S(`" class="align-middle" hx-target="this" hx-swap="outerHTML">
+	qw422016.N().S(`" class="align-middle alert alert-warning" hx-target="this" hx-swap="outerHTML">
     <th scope="row"><input
-            id="`)
+            id="input-`)
 	qw422016.N().S(id)
-	qw422016.N().S(`-name-input"
+	qw422016.N().S(`-name"
             class="form-control form-control-sm"
             type="text"
             name="name"
             placeholder="Name"
     ></th>
     <td><input
-            id="`)
+            id="input-`)
 	qw422016.N().S(id)
-	qw422016.N().S(`-url-input"
+	qw422016.N().S(`-url"
             class="form-control form-control-sm"
             type="url"
             name="url"
@@ -314,11 +314,11 @@ func StreamRenderNewFeedItemRow(qw422016 *qt422016.Writer) {
                     hx-post="`)
 	qw422016.N().S(urls.NewFeedItem)
 	qw422016.N().S(`"
-                    hx-include="#`)
+                    hx-include="#input-`)
 	qw422016.N().S(id)
-	qw422016.N().S(`-name-input, #`)
+	qw422016.N().S(`-name, #input-`)
 	qw422016.N().S(id)
-	qw422016.N().S(`-url-input">
+	qw422016.N().S(`-url">
                 <i class="bi bi-check"></i>
             </button>
             <button type="button" class="btn btn-outline-danger" id="`)
diff --git a/walrss/internal/rss/processor.go b/walrss/internal/rss/processor.go
new file mode 100644
index 0000000..e220f0a
--- /dev/null
+++ b/walrss/internal/rss/processor.go
@@ -0,0 +1,187 @@
+package rss
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"github.com/carlmjohnson/requests"
+	"github.com/codemicro/walrss/walrss/internal/core"
+	"github.com/codemicro/walrss/walrss/internal/db"
+	"github.com/codemicro/walrss/walrss/internal/state"
+	"github.com/matcornic/hermes"
+	"github.com/mmcdole/gofeed"
+	"github.com/patrickmn/go-cache"
+	"io/ioutil"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	dateFormat = "02Jan06"
+	timeFormat = "15:04:05"
+)
+
+type processedFeed struct {
+	Name  string
+	Items []*feedItem
+	Error error
+}
+
+func ProcessFeeds(st *state.State, day db.SendDay, hour int) error {
+	u, e := core.GetUsersBySchedule(st, day, hour)
+	for _, ur := range u {
+		fmt.Printf("%#v\n", ur)
+
+		userFeeds, err := core.GetFeedsForUser(st, ur.ID)
+		if err != nil {
+			return err
+		}
+
+		var processedFeeds []*processedFeed
+
+		for _, f := range userFeeds {
+			pf := new(processedFeed)
+			pf.Name = f.Name
+
+			rawFeed, err := getFeedContent(f.URL)
+			if err != nil {
+				pf.Error = err
+			} else {
+				pf.Items = filterFeedContent(rawFeed, time.Date(2022, 04, 01, 0, 0, 0, 0, time.UTC))
+			}
+			processedFeeds = append(processedFeeds, pf)
+		}
+
+		plainContent, htmlContent, err := generateEmail(processedFeeds)
+		if err != nil {
+			return err
+		}
+
+		// TODO: Send email
+	}
+
+	return nil
+}
+
+var (
+	feedCache     = cache.New(time.Minute*10, time.Minute*20)
+	feedFetchLock = new(sync.Mutex)
+)
+
+func getFeedContent(url string) (*gofeed.Feed, error) {
+	feedFetchLock.Lock()
+	defer feedFetchLock.Unlock()
+
+	if v, found := feedCache.Get(url); found {
+		return v.(*gofeed.Feed), nil
+	}
+
+	buf := new(bytes.Buffer)
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+	defer cancel()
+
+	if err := requests.URL(url).ToBytesBuffer(buf).Fetch(ctx); err != nil {
+		return nil, err
+	}
+
+	feed, err := gofeed.NewParser().Parse(buf)
+	if err != nil {
+		return nil, err
+	}
+
+	_ = feedCache.Add(url, feed, cache.DefaultExpiration)
+
+	return feed, nil
+}
+
+type feedItem struct {
+	Title       string
+	URL         string
+	PublishTime time.Time
+}
+
+func filterFeedContent(feed *gofeed.Feed, earliestPublishTime time.Time) []*feedItem {
+	var o []*feedItem
+
+	for _, item := range feed.Items {
+		if item.PublishedParsed != nil && item.PublishedParsed.After(earliestPublishTime) {
+			o = append(o, &feedItem{
+				Title:       item.Title,
+				URL:         item.Link,
+				PublishTime: *item.PublishedParsed,
+			})
+		}
+	}
+
+	return o
+}
+
+func generateEmail(processedItems []*processedFeed) (plain, html []byte, err error) {
+	sort.Slice(processedItems, func(i, j int) bool {
+		pi, pj := processedItems[i], processedItems[j]
+
+		if pi.Error != nil && pj.Error == nil {
+			return false
+		}
+
+		if pi.Error == nil && pj.Error != nil {
+			return true
+		}
+
+		return pi.Name < pj.Name
+	})
+
+	var sb strings.Builder
+
+	for _, processedItem := range processedItems {
+
+		if len(processedItem.Items) != 0 || processedItem.Error != nil {
+			sb.WriteString("* **")
+			sb.WriteString(strings.ReplaceAll(processedItem.Name, "*", `\*`))
+			sb.WriteString("**\n")
+		}
+
+		if processedItem.Error != nil {
+			sb.WriteString("  * **Error:** ")
+			sb.WriteString(processedItem.Error.Error())
+			sb.WriteString("\n")
+		} else {
+			r := strings.NewReplacer("[", `\[`, "]", `\]`, "*", `\*`)
+
+			for _, item := range processedItem.Items {
+				sb.WriteString("  * [**")
+				sb.WriteString(r.Replace(item.Title))
+				sb.WriteString("**](")
+				sb.WriteString(item.URL)
+				sb.WriteString(") - ")
+				sb.WriteString(strings.ToUpper(item.PublishTime.Format(dateFormat + " " + timeFormat)))
+				sb.WriteString("\n")
+			}
+
+		}
+	}
+
+	e := hermes.Email{
+		Body: hermes.Body{
+			FreeMarkdown: hermes.Markdown(sb.String()),
+		},
+	}
+
+	renderer := hermes.Hermes{
+		Theme: new(hermes.Flat),
+	}
+
+	plainString, err := renderer.GeneratePlainText(e)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	htmlString, err := renderer.GenerateHTML(e)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return []byte(plainString), []byte(htmlString), nil
+}
diff --git a/walrss/internal/rss/watcher.go b/walrss/internal/rss/watcher.go
new file mode 100644
index 0000000..e7036c1
--- /dev/null
+++ b/walrss/internal/rss/watcher.go
@@ -0,0 +1,26 @@
+package rss
+
+import (
+	"github.com/codemicro/walrss/walrss/internal/db"
+	"github.com/codemicro/walrss/walrss/internal/state"
+	"github.com/rs/zerolog/log"
+	"time"
+)
+
+func StartWatcher(st *state.State) {
+	go func() {
+		currentTime := time.Now().UTC()
+		time.Sleep(time.Minute * time.Duration(60-currentTime.Minute()))
+
+		if err := ProcessFeeds(st, db.SendDayFromWeekday(currentTime.Weekday()), currentTime.Hour()+1); err != nil {
+			log.Error().Err(err).Str("location", "feed watcher").Send()
+		}
+
+		ticker := time.NewTicker(time.Hour)
+		for currentTime := range ticker.C {
+			if err := ProcessFeeds(st, db.SendDayFromWeekday(currentTime.Weekday()), currentTime.Hour()); err != nil {
+				log.Error().Err(err).Str("location", "feed watcher").Send()
+			}
+		}
+	}()
+}
diff --git a/walrss/internal/state/state.go b/walrss/internal/state/state.go
index 54badd7..31cb59b 100644
--- a/walrss/internal/state/state.go
+++ b/walrss/internal/state/state.go
@@ -19,6 +19,13 @@ func New() *State {
 }
 
 type Config struct {
+	//Email struct {
+	//	Host     string `fig:"host" validate:"required"`
+	//	Username string `fig:"username" validate:"required"`
+	//	Password string `fig:"password" validate:"required"`
+	//	From     string `fig:"from" validate:"required"`
+	//	Port     int    `fig:"port" validate:"required"`
+	//}
 	Server struct {
 		Host string `fig:"host" default:"127.0.0.1"`
 		Port int    `fig:"port" default:"8080"`
diff --git a/walrss/main.go b/walrss/main.go
index 9cb951d..9978bab 100644
--- a/walrss/main.go
+++ b/walrss/main.go
@@ -3,6 +3,7 @@ package main
 import (
 	"github.com/codemicro/walrss/walrss/internal/db"
 	"github.com/codemicro/walrss/walrss/internal/http"
+	"github.com/codemicro/walrss/walrss/internal/rss"
 	"github.com/codemicro/walrss/walrss/internal/state"
 	"github.com/rs/zerolog/log"
 	"os"
@@ -38,6 +39,8 @@ func run() error {
 		return err
 	}
 
+	rss.ProcessFeeds(st, db.SendOnSunday, 21)
+
 	return server.Run()
 }