adulau/mastodon-markdown-archive
1
0
Fork 0
mirror of https://github.com/adulau/mastodon-markdown-archive.git synced 2024-11-22 01:47:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Clean up some leaky abstractions

Browse source
This commit is contained in:
Gabriel Garrido 2024-05-18 15:22:23 +02:00
parent 437db12b8d
commit f97a0e47af
4 changed files with 50 additions and 581 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

277
client/client.go
View file

@ -1,222 +1,3 @@
<<<<<<< Updated upstream
package client
import (
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
type Client struct {
handle string
baseURL string
}
type Account struct {
Id string `json:"id"`
Username string `json:"username"`
Acct string `json:"acct"`
DisplayName string `json:"display_name"`
Locked bool `json:"locked"`
Bot bool `json:"bot"`
Discoverable bool `json:"discoverable"`
Group bool `json:"group"`
CreatedAt time.Time `json:"created_at"`
Note string `json:"note"`
URL string `json:"url"`
URI string `json:"uri"`
Avatar string `json:"avatar"`
AvatarStatic string `json:"avatar_static"`
Header string `json:"header"`
HeaderStatic string `json:"header_static"`
FollowersCount int `json:"followers_count"`
FollowingCount int `json:"following_count"`
StatusesCount int `json:"statuses_count"`
LastStatusAt string `json:"last_status_at"`
}
type MediaAttachment struct {
Type string `json:"type"`
URL string `json:"url"`
Description string `json:"description"`
Id string `json:"id"`
Path string
}
type Application struct {
Name string `json:"name"`
Website string `json:"website"`
}
type Tag struct {
Name string `json:"name"`
URL string `json:"url"`
}
type Post struct {
CreatedAt time.Time `json:"created_at"`
Id string `json:"id"`
Visibility string `json:"visibility"`
InReplyToId string `json:"in_reply_to_id"`
InReplyToAccountId string `json:"in_reply_to_account_id"`
Sensitive bool `json:"sensitive"`
SpoilerText string `json:"spoiler_text"`
Language string `json:"language"`
URI string `json:"uri"`
URL string `json:"url"`
Application Application `json:"application"`
Content string `json:"content"`
MediaAttachments []MediaAttachment `json:"media_attachments"`
RepliesCount int `json:"replies_count"`
ReblogsCount int `json:"reblogs_count"`
FavoritesCount int `json:"favourites_count"`
Pinned bool `json:"pinned"`
Tags []Tag `json:"tags"`
Favourited bool `json:"favourited"`
Reblogged bool `json:"reblogged"`
Muted bool `json:"muted"`
Bookmarked bool `json:"bookmarked"`
Account Account `json:"account"`
}
type PostsFilter struct {
ExcludeReplies bool
ExcludeReblogs bool
Limit int
SinceId string
MinId string
MaxId string
}
func New(userURL string) (Client, error) {
var client Client
parsedURL, err := url.Parse(userURL)
if err != nil {
return client, fmt.Errorf("error parsing user url: %w", err)
}
baseURL := fmt.Sprintf("%s://%s", parsedURL.Scheme, parsedURL.Host)
acc := strings.TrimPrefix(parsedURL.Path, "/")
handle := strings.TrimPrefix(acc, "@")
return Client{
baseURL: baseURL,
handle: handle,
}, nil
}
func (c Client) Posts(filter PostsFilter) ([]Post, error) {
var posts []Post
account, err := c.getAccount()
if err != nil {
return posts, err
}
queryValues := url.Values{}
if filter.ExcludeReplies {
queryValues.Add("exclude_replies", strconv.Itoa(1))
}
if filter.ExcludeReblogs {
queryValues.Add("exclude_reblogs", strconv.Itoa(1))
}
if filter.SinceId != "" {
queryValues.Add("since_id", filter.SinceId)
}
if filter.MaxId != "" {
queryValues.Add("max_id", filter.MaxId)
}
if filter.MinId != "" {
queryValues.Add("min_id", filter.MinId)
}
queryValues.Add("limit", strconv.Itoa(filter.Limit))
query := fmt.Sprintf("?%s", queryValues.Encode())
postsUrl := fmt.Sprintf(
"%s/api/v1/accounts/%s/statuses/%s",
c.baseURL,
account.Id,
query,
)
log.Println(fmt.Sprintf("Fetching posts from %s", postsUrl))
if err := get(postsUrl, &posts); err != nil {
return posts, err
}
return posts, nil
}
func (c Client) getAccount() (Account, error) {
var account Account
lookupUrl := fmt.Sprintf(
"%s/api/v1/accounts/lookup?acct=%s",
c.baseURL,
c.handle,
)
err := get(lookupUrl, &account)
if err != nil {
return account, err
}
return account, nil
}
func TagsForPost(post Post, descendants []Post) []Tag {
var tags []Tag
for _, tag := range post.Tags {
tags = append(tags, tag)
}
for _, descendant := range descendants {
for _, tag := range descendant.Tags {
tags = append(tags, tag)
}
}
return tags
}
func get(requestUrl string, variable interface{}) error {
res, err := http.Get(requestUrl)
if err != nil {
return err
}
defer res.Body.Close()
body, err := io.ReadAll(res.Body)
if err := json.Unmarshal(body, variable); err != nil {
return err
}
return nil
}
func ShouldSkipPost(post Post) bool {
return post.Visibility != "public"
}
=======
package client package client
import ( import (
@ -237,9 +18,10 @@ type Client struct {
filters PostsFilter filters PostsFilter
account Account account Account
posts []Post posts []Post
replies map[string]Post replies map[string]string
orphans []string orphans []string
postIdMap map[string]Post postIdMap map[string]int
output []int
} }
type Account struct { type Account struct {
@ -307,7 +89,7 @@ type Post struct {
Muted bool `json:"muted"` Muted bool `json:"muted"`
Bookmarked bool `json:"bookmarked"` Bookmarked bool `json:"bookmarked"`
Account Account `json:"account"` Account Account `json:"account"`
descendants []Post descendants []*Post
} }
type PostsFilter struct { type PostsFilter struct {
@ -350,8 +132,8 @@ func New(userURL string, filters PostsFilter, threaded bool) (Client, error) {
filters: filters, filters: filters,
account: account, account: account,
posts: posts, posts: posts,
postIdMap: make(map[string]Post), postIdMap: make(map[string]int),
replies: make(map[string]Post), replies: make(map[string]string),
orphans: orphans, orphans: orphans,
} }
@ -359,10 +141,10 @@ func New(userURL string, filters PostsFilter, threaded bool) (Client, error) {
if threaded { if threaded {
client.generateReplies() client.generateReplies()
} else {
for i := range client.posts {
client.output = append(client.output, i)
} }
for _, orphan := range client.orphans {
log.Println(fmt.Sprintf("Orphan: %s", orphan))
} }
return client, nil return client, nil
@ -372,43 +154,51 @@ func (c Client) Account() Account {
return c.account return c.account
} }
func (c Client) Posts() []Post { func (c Client) Posts() []*Post {
return c.posts var p []*Post
for _, i := range c.output {
p = append(p, &c.posts[i])
}
return p
} }
func (p Post) ShouldSkip() bool { func (p Post) ShouldSkip() bool {
return p.Visibility != "public" return p.Visibility != "public"
} }
func (p Post) Descendants() []Post { func (p Post) Descendants() []*Post {
return p.descendants return p.descendants
} }
func (c Client) populateIdMap() { func (c *Client) populateIdMap() {
for _, post := range c.posts { for i, post := range c.posts {
c.postIdMap[post.Id] = post c.postIdMap[post.Id] = i
} }
} }
func (c Client) flushReplies(post Post, descendants *[]Post) { func (c *Client) flushReplies(post *Post, descendants *[]*Post) {
if reply, ok := c.replies[post.Id]; ok { if pid, ok := c.replies[post.Id]; ok {
*descendants = append(*descendants, reply) reply := c.posts[c.postIdMap[pid]]
c.flushReplies(reply, descendants) *descendants = append(*descendants, &reply)
c.flushReplies(&reply, descendants)
} }
} }
func (c Client) generateReplies() { func (c *Client) generateReplies() {
for _, post := range c.posts { for i := range c.posts {
post := &c.posts[i]
if post.InReplyToId == "" { if post.InReplyToId == "" {
c.flushReplies(post, &post.descendants) c.flushReplies(post, &post.descendants)
c.output = append(c.output, i)
continue continue
} }
if _, ok := c.postIdMap[post.Id]; ok { if _, ok := c.postIdMap[post.InReplyToId]; ok {
log.Println(fmt.Sprintf("Adding %s to replies of %s", post.Id, post.InReplyToId)) // TODO: Exclude from list of posts that gets rendered to disc
c.replies[post.InReplyToId] = post c.replies[post.InReplyToId] = post.Id
} else { } else {
log.Println("Found orphan")
c.orphans = append(c.orphans, post.Id) c.orphans = append(c.orphans, post.Id)
} }
} }
@ -525,4 +315,3 @@ func get(requestUrl string, variable interface{}) error {
return nil return nil
} }
>>>>>>> Stashed changes

204
files/files.go
View file

@ -1,200 +1,3 @@
<<<<<<< Updated upstream
package files
import (
"embed"
"fmt"
"io"
"mime"
"net/http"
"os"
"path/filepath"
"text/template"
"git.garrido.io/gabriel/mastodon-markdown-archive/client"
md "github.com/JohannesKaufmann/html-to-markdown"
)
//go:embed templates/post.tmpl
var templates embed.FS
type FileWriter struct {
dir string
repies map[string]client.Post
}
type TemplateContext struct {
Post client.Post
Descendants []client.Post
Tags []client.Tag
}
func New(dir string) (FileWriter, error) {
var fileWriter FileWriter
_, err := os.Stat(dir)
if os.IsNotExist(err) {
os.Mkdir(dir, os.ModePerm)
}
absDir, err := filepath.Abs(dir)
if err != nil {
return fileWriter, err
}
return FileWriter{
dir: absDir,
repies: make(map[string]client.Post),
}, nil
}
func (f FileWriter) Write(post client.Post, threaded bool, templateFile string) error {
if threaded && post.InReplyToId != "" {
f.repies[post.InReplyToId] = post
return nil
}
var descendants []client.Post
f.getReplies(post.Id, &descendants)
var file *os.File
var err error
if len(post.MediaAttachments) == 0 {
name := fmt.Sprintf("%s.md", post.Id)
filename := filepath.Join(f.dir, name)
file, err = os.Create(filename)
} else {
dir := filepath.Join(f.dir, post.Id)
os.Mkdir(dir, os.ModePerm)
for i := 0; i < len(post.MediaAttachments); i++ {
media := &post.MediaAttachments[i]
if media.Type != "image" {
continue
}
imageFilename, err := downloadAttachment(dir, media.Id, media.URL)
if err != nil {
return err
}
media.Path = imageFilename
}
filename := filepath.Join(dir, "index.md")
file, err = os.Create(filename)
}
if err != nil {
return fmt.Errorf("error creating file: %w", err)
}
defer file.Close()
tmpl, err := resolveTemplate(templateFile)
context := TemplateContext{
Post: post,
Descendants: descendants,
Tags: client.TagsForPost(post, descendants),
}
err = tmpl.Execute(file, context)
if err != nil {
return fmt.Errorf("error executing template: %w", err)
}
return nil
}
func (f FileWriter) getReplies(postId string, replies *[]client.Post) {
if reply, ok := f.repies[postId]; ok {
*replies = append(*replies, reply)
f.getReplies(reply.Id, replies)
}
}
func downloadAttachment(dir string, id string, url string) (string, error) {
var filename string
client := &http.Client{}
req, _ := http.NewRequest("GET", url, nil)
req.Header.Set("Accept", "image/*")
res, err := client.Do(req)
if err != nil {
return filename, err
}
defer res.Body.Close()
contentType := res.Header.Get("Content-Type")
extensions, err := mime.ExtensionsByType(contentType)
if err != nil {
return filename, err
}
var extension string
urlExtension := filepath.Ext(url)
for _, i := range extensions {
if i == urlExtension {
extension = i
break
}
}
if extension == "" {
return filename, fmt.Errorf("could not match extension for media")
}
filename = fmt.Sprintf("%s%s", id, extension)
file, err := os.Create(filepath.Join(dir, filename))
if err != nil {
return filename, err
}
defer file.Close()
_, err = io.Copy(file, res.Body)
if err != nil {
return filename, err
}
return filename, nil
}
func resolveTemplate(templateFile string) (*template.Template, error) {
converter := md.NewConverter("", true, nil)
funcs := template.FuncMap{
"tomd": converter.ConvertString,
}
if templateFile == "" {
tmpl, err := template.New("post.tmpl").Funcs(funcs).ParseFS(templates, "templates/*.tmpl")
if err != nil {
return tmpl, err
}
return tmpl, nil
}
tmpl, err := template.New(filepath.Base(templateFile)).Funcs(funcs).ParseGlob(templateFile)
if err != nil {
return tmpl, err
}
return tmpl, nil
}
=======
package files package files
import ( import (
@ -219,7 +22,7 @@ type FileWriter struct {
} }
type TemplateContext struct { type TemplateContext struct {
Post client.Post Post *client.Post
} }
type PostFile struct { type PostFile struct {
@ -247,7 +50,7 @@ func New(dir string) (FileWriter, error) {
}, nil }, nil
} }
func (f FileWriter) Write(post client.Post, templateFile string) error { func (f FileWriter) Write(post *client.Post, templateFile string) error {
hasMedia := len(post.AllMedia()) > 0 hasMedia := len(post.AllMedia()) > 0
postFile, err := f.createFile(post, hasMedia) postFile, err := f.createFile(post, hasMedia)
@ -286,7 +89,7 @@ func (f FileWriter) Write(post client.Post, templateFile string) error {
return nil return nil
} }
func (f FileWriter) createFile(post client.Post, shouldBundle bool) (PostFile, error) { func (f FileWriter) createFile(post *client.Post, shouldBundle bool) (PostFile, error) {
var postFile PostFile var postFile PostFile
if shouldBundle { if shouldBundle {
@ -425,4 +228,3 @@ func resolveTemplate(templateFile string) (*template.Template, error) {
return tmpl, nil return tmpl, nil
} }
>>>>>>> Stashed changes

16
files/templates/post.tmpl
View file

@ -19,28 +19,13 @@ descendants:
{{- end }} {{- end }}
--- ---
{{ .Post.Content | tomd }} {{ .Post.Content | tomd }}
<<<<<<< Updated upstream
{{- range .Post.MediaAttachments }}
{{ if eq .Type "image" }}
=======
{{ range .Post.MediaAttachments }} {{ range .Post.MediaAttachments }}
{{- if eq .Type "image" }} {{- if eq .Type "image" }}
>>>>>>> Stashed changes
![{{ .Description }}]({{ .Path }}) ![{{ .Description }}]({{ .Path }})
{{ end }} {{ end }}
{{- end -}} {{- end -}}
<<<<<<< Updated upstream
{{- range .Descendants }}
{{ .Content | tomd -}}
{{- range .MediaAttachments }}
{{ if eq .Type "image" }}
![{{ .Description }}]({{ .Path }})
{{ end }}
{{- end -}}
{{- end -}}
=======
{{ range .Post.Descendants }} {{ range .Post.Descendants }}
{{ .Content | tomd }} {{ .Content | tomd }}
{{ range .MediaAttachments }} {{ range .MediaAttachments }}
@ -49,4 +34,3 @@ descendants:
{{- end }} {{- end }}
{{- end }} {{- end }}
{{- end }} {{- end }}
>>>>>>> Stashed changes

106
main.go
View file

@ -1,108 +1,3 @@
<<<<<<< Updated upstream
package main
import (
"flag"
"fmt"
"log"
"os"
"path/filepath"
"git.garrido.io/gabriel/mastodon-markdown-archive/client"
"git.garrido.io/gabriel/mastodon-markdown-archive/files"
)
func main() {
dist := flag.String("dist", "./posts", "Path to directory where files will be written")
user := flag.String("user", "", "URL of User's Mastodon account whose toots will be fetched")
excludeReplies := flag.Bool("exclude-replies", false, "Whether or not exclude replies to other users")
excludeReblogs := flag.Bool("exclude-reblogs", false, "Whether or not to exclude reblogs")
limit := flag.Int("limit", 40, "Maximum number of posts to fetch")
sinceId := flag.String("since-id", "", "Fetch posts greater than this id")
maxId := flag.String("max-id", "", "Fetch posts lesser than this id")
minId := flag.String("min-id", "", "Fetch posts immediately newer than this id")
persistFirst := flag.String("persist-first", "", "Location to persist the post id of the first post returned")
persistLast := flag.String("persist-last", "", "Location to persist the post id of the last post returned")
templateFile := flag.String("template", "", "Template to use for post rendering, if passed")
threaded := flag.Bool("threaded", true, "Thread replies for a post in a single file")
flag.Parse()
c, err := client.New(*user)
if err != nil {
log.Panicln(fmt.Errorf("error instantiating client: %w", err))
}
posts, err := c.Posts(client.PostsFilter{
ExcludeReplies: *excludeReplies,
ExcludeReblogs: *excludeReblogs,
Limit: *limit,
SinceId: *sinceId,
MaxId: *maxId,
MinId: *minId,
})
if err != nil {
log.Panicln(err)
}
fileWriter, err := files.New(*dist)
if err != nil {
log.Panicln(err)
}
log.Println(fmt.Sprintf("Fetched %d posts", len(posts)))
for _, post := range posts {
if client.ShouldSkipPost(post) {
continue
}
if err := fileWriter.Write(post, *threaded, *templateFile); err != nil {
log.Panicln("error writing post to file: %w", err)
break
}
}
postsCount := len(posts)
if postsCount > 0 {
if *persistFirst != "" {
firstPost := posts[0]
err := persistId(firstPost.Id, *persistFirst)
if err != nil {
log.Panicln(err)
}
}
if *persistLast != "" {
lastPost := posts[postsCount-1]
err := persistId(lastPost.Id, *persistLast)
if err != nil {
log.Panicln(err)
}
}
}
}
func persistId(postId string, path string) error {
persistPath, err := filepath.Abs(path)
if err != nil {
return err
}
if err := os.WriteFile(persistPath, []byte(postId), 0644); err != nil {
return err
}
return nil
}
=======
package main package main
import ( import (
@ -196,4 +91,3 @@ func persistId(postId string, path string) error {
return nil return nil
} }
>>>>>>> Stashed changes