aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAKP <abi@tdpain.net>2025-02-10 22:59:03 +0000
committerAKP <abi@tdpain.net>2025-02-10 22:59:03 +0000
commitb95ae4d4bd44b48955940e822f1556e0f8623916 (patch)
treedde4a35c66b14651b87034c2dce83c412229c0bd
parent83a0816f8cc90f3f00d2c4918d959c37da915347 (diff)
Use batch insert when loading OPML
This is many many times faster (very noticeably) than using one query per feed as it was before.
-rw-r--r--CHANGELOG.md3
-rw-r--r--walrss/internal/core/feeds.go37
2 files changed, 32 insertions, 8 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
index aab606a..4c2e1dd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,7 +8,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
### Changed
* When no feed items are stored in the database for a particular feed, fall back to using the old time-based detection method
* Do not index feed items at addition time
-* Do not update feed item index when sending a test email to prevent items being missed from the main scheduled emails
+* Do not update feed item index when sending a test email to prevent items being missed from the main scheduled emails
+* Use batch inserts when importing from OPML
## 0.4.2 - 2025-02-09
### Fixed
diff --git a/walrss/internal/core/feeds.go b/walrss/internal/core/feeds.go
index 63110f5..0cb5fdf 100644
--- a/walrss/internal/core/feeds.go
+++ b/walrss/internal/core/feeds.go
@@ -33,6 +33,26 @@ func NewFeed(st *state.State, userID, name, url string) (*db.Feed, error) {
return feed, nil
}
+func NewFeeds(st *state.State, userID string, fs []*db.Feed) error {
+ if len(fs) == 0 {
+ return nil
+ }
+
+ for i, f := range fs {
+ f.ID = shortuuid.New()
+ f.UserID = userID
+ if err := validateFeedName(f.Name); err != nil {
+ return NewUserErrorWithStatus(400, "validate feed %d: %w", i, err)
+ }
+ if err := validateURL(f.URL); err != nil {
+ return NewUserErrorWithStatus(400, "validate feed %d: %w", i, err)
+ }
+ }
+
+ _, err := st.Data.NewInsert().Model(&fs).Exec(context.Background())
+ return err
+}
+
func GetFeedsForUser(st *state.State, userID string) (res []*db.Feed, err error) {
err = st.Data.NewSelect().
Model(&res).
@@ -110,14 +130,17 @@ func ImportFeedsForUser(st *state.State, userID string, opmlXML []byte) error {
}
}
- for _, feed := range o.ToFeeds() {
- if _, found := existingURLs[feed.URL]; found {
- continue
- }
- if _, err := NewFeed(st, userID, feed.Name, feed.URL); err != nil {
- return err
+ var (
+ fs = o.ToFeeds()
+ n int
+ )
+ for _, feed := range fs {
+ if _, found := existingURLs[feed.URL]; !found {
+ fs[n] = feed
+ n += 1
}
}
+ fs = fs[:n]
- return nil
+ return NewFeeds(st, userID, fs)
}