From f97a0e47af04d19d0d189b226c26904198fb50c8 Mon Sep 17 00:00:00 2001 From: Gabriel Garrido Date: Sat, 18 May 2024 15:22:23 +0200 Subject: [PATCH] Clean up some leaky abstractions --- client/client.go | 291 ++++++-------------------------------- files/files.go | 204 +------------------------- files/templates/post.tmpl | 16 --- main.go | 120 +--------------- 4 files changed, 50 insertions(+), 581 deletions(-) diff --git a/client/client.go b/client/client.go index 49f5f39..e3cd576 100644 --- a/client/client.go +++ b/client/client.go @@ -1,222 +1,3 @@ -<<<<<<< Updated upstream -package client - -import ( - "encoding/json" - "fmt" - "io" - "log" - "net/http" - "net/url" - "strconv" - "strings" - "time" -) - -type Client struct { - handle string - baseURL string -} - -type Account struct { - Id string `json:"id"` - Username string `json:"username"` - Acct string `json:"acct"` - DisplayName string `json:"display_name"` - Locked bool `json:"locked"` - Bot bool `json:"bot"` - Discoverable bool `json:"discoverable"` - Group bool `json:"group"` - CreatedAt time.Time `json:"created_at"` - Note string `json:"note"` - URL string `json:"url"` - URI string `json:"uri"` - Avatar string `json:"avatar"` - AvatarStatic string `json:"avatar_static"` - Header string `json:"header"` - HeaderStatic string `json:"header_static"` - FollowersCount int `json:"followers_count"` - FollowingCount int `json:"following_count"` - StatusesCount int `json:"statuses_count"` - LastStatusAt string `json:"last_status_at"` -} - -type MediaAttachment struct { - Type string `json:"type"` - URL string `json:"url"` - Description string `json:"description"` - Id string `json:"id"` - Path string -} - -type Application struct { - Name string `json:"name"` - Website string `json:"website"` -} - -type Tag struct { - Name string `json:"name"` - URL string `json:"url"` -} - -type Post struct { - CreatedAt time.Time `json:"created_at"` - Id string `json:"id"` - Visibility string `json:"visibility"` - InReplyToId string `json:"in_reply_to_id"` - InReplyToAccountId string `json:"in_reply_to_account_id"` - Sensitive bool `json:"sensitive"` - SpoilerText string `json:"spoiler_text"` - Language string `json:"language"` - URI string `json:"uri"` - URL string `json:"url"` - Application Application `json:"application"` - Content string `json:"content"` - MediaAttachments []MediaAttachment `json:"media_attachments"` - RepliesCount int `json:"replies_count"` - ReblogsCount int `json:"reblogs_count"` - FavoritesCount int `json:"favourites_count"` - Pinned bool `json:"pinned"` - Tags []Tag `json:"tags"` - Favourited bool `json:"favourited"` - Reblogged bool `json:"reblogged"` - Muted bool `json:"muted"` - Bookmarked bool `json:"bookmarked"` - Account Account `json:"account"` -} - -type PostsFilter struct { - ExcludeReplies bool - ExcludeReblogs bool - Limit int - SinceId string - MinId string - MaxId string -} - -func New(userURL string) (Client, error) { - var client Client - parsedURL, err := url.Parse(userURL) - - if err != nil { - return client, fmt.Errorf("error parsing user url: %w", err) - } - - baseURL := fmt.Sprintf("%s://%s", parsedURL.Scheme, parsedURL.Host) - acc := strings.TrimPrefix(parsedURL.Path, "/") - handle := strings.TrimPrefix(acc, "@") - - return Client{ - baseURL: baseURL, - handle: handle, - }, nil -} - -func (c Client) Posts(filter PostsFilter) ([]Post, error) { - var posts []Post - account, err := c.getAccount() - - if err != nil { - return posts, err - } - - queryValues := url.Values{} - - if filter.ExcludeReplies { - queryValues.Add("exclude_replies", strconv.Itoa(1)) - } - - if filter.ExcludeReblogs { - queryValues.Add("exclude_reblogs", strconv.Itoa(1)) - } - - if filter.SinceId != "" { - queryValues.Add("since_id", filter.SinceId) - } - - if filter.MaxId != "" { - queryValues.Add("max_id", filter.MaxId) - } - - if filter.MinId != "" { - queryValues.Add("min_id", filter.MinId) - } - - queryValues.Add("limit", strconv.Itoa(filter.Limit)) - - query := fmt.Sprintf("?%s", queryValues.Encode()) - - postsUrl := fmt.Sprintf( - "%s/api/v1/accounts/%s/statuses/%s", - c.baseURL, - account.Id, - query, - ) - - log.Println(fmt.Sprintf("Fetching posts from %s", postsUrl)) - - if err := get(postsUrl, &posts); err != nil { - return posts, err - } - - return posts, nil -} - -func (c Client) getAccount() (Account, error) { - var account Account - lookupUrl := fmt.Sprintf( - "%s/api/v1/accounts/lookup?acct=%s", - c.baseURL, - c.handle, - ) - - err := get(lookupUrl, &account) - - if err != nil { - return account, err - } - - return account, nil -} - -func TagsForPost(post Post, descendants []Post) []Tag { - var tags []Tag - - for _, tag := range post.Tags { - tags = append(tags, tag) - } - - for _, descendant := range descendants { - for _, tag := range descendant.Tags { - tags = append(tags, tag) - } - } - - return tags -} - -func get(requestUrl string, variable interface{}) error { - res, err := http.Get(requestUrl) - - if err != nil { - return err - } - - defer res.Body.Close() - - body, err := io.ReadAll(res.Body) - - if err := json.Unmarshal(body, variable); err != nil { - return err - } - - return nil -} - -func ShouldSkipPost(post Post) bool { - return post.Visibility != "public" -} -======= package client import ( @@ -237,9 +18,10 @@ type Client struct { filters PostsFilter account Account posts []Post - replies map[string]Post + replies map[string]string orphans []string - postIdMap map[string]Post + postIdMap map[string]int + output []int } type Account struct { @@ -307,7 +89,7 @@ type Post struct { Muted bool `json:"muted"` Bookmarked bool `json:"bookmarked"` Account Account `json:"account"` - descendants []Post + descendants []*Post } type PostsFilter struct { @@ -345,24 +127,24 @@ func New(userURL string, filters PostsFilter, threaded bool) (Client, error) { var orphans []string client = Client{ - baseURL: baseURL, - handle: handle, - filters: filters, - account: account, - posts: posts, - postIdMap: make(map[string]Post), - replies: make(map[string]Post), - orphans: orphans, + baseURL: baseURL, + handle: handle, + filters: filters, + account: account, + posts: posts, + postIdMap: make(map[string]int), + replies: make(map[string]string), + orphans: orphans, } client.populateIdMap() if threaded { client.generateReplies() - } - - for _, orphan := range client.orphans { - log.Println(fmt.Sprintf("Orphan: %s", orphan)) + } else { + for i := range client.posts { + client.output = append(client.output, i) + } } return client, nil @@ -372,43 +154,51 @@ func (c Client) Account() Account { return c.account } -func (c Client) Posts() []Post { - return c.posts +func (c Client) Posts() []*Post { + var p []*Post + + for _, i := range c.output { + p = append(p, &c.posts[i]) + } + + return p } func (p Post) ShouldSkip() bool { return p.Visibility != "public" } -func (p Post) Descendants() []Post { +func (p Post) Descendants() []*Post { return p.descendants } -func (c Client) populateIdMap() { - for _, post := range c.posts { - c.postIdMap[post.Id] = post +func (c *Client) populateIdMap() { + for i, post := range c.posts { + c.postIdMap[post.Id] = i } } -func (c Client) flushReplies(post Post, descendants *[]Post) { - if reply, ok := c.replies[post.Id]; ok { - *descendants = append(*descendants, reply) - c.flushReplies(reply, descendants) +func (c *Client) flushReplies(post *Post, descendants *[]*Post) { + if pid, ok := c.replies[post.Id]; ok { + reply := c.posts[c.postIdMap[pid]] + *descendants = append(*descendants, &reply) + c.flushReplies(&reply, descendants) } } -func (c Client) generateReplies() { - for _, post := range c.posts { +func (c *Client) generateReplies() { + for i := range c.posts { + post := &c.posts[i] if post.InReplyToId == "" { c.flushReplies(post, &post.descendants) + c.output = append(c.output, i) continue } - if _, ok := c.postIdMap[post.Id]; ok { - log.Println(fmt.Sprintf("Adding %s to replies of %s", post.Id, post.InReplyToId)) - c.replies[post.InReplyToId] = post + if _, ok := c.postIdMap[post.InReplyToId]; ok { + // TODO: Exclude from list of posts that gets rendered to disc + c.replies[post.InReplyToId] = post.Id } else { - log.Println("Found orphan") c.orphans = append(c.orphans, post.Id) } } @@ -525,4 +315,3 @@ func get(requestUrl string, variable interface{}) error { return nil } ->>>>>>> Stashed changes diff --git a/files/files.go b/files/files.go index 2421cb2..fb66573 100644 --- a/files/files.go +++ b/files/files.go @@ -1,200 +1,3 @@ -<<<<<<< Updated upstream -package files - -import ( - "embed" - "fmt" - "io" - "mime" - "net/http" - "os" - "path/filepath" - "text/template" - - "git.garrido.io/gabriel/mastodon-markdown-archive/client" - md "github.com/JohannesKaufmann/html-to-markdown" -) - -//go:embed templates/post.tmpl -var templates embed.FS - -type FileWriter struct { - dir string - repies map[string]client.Post -} - -type TemplateContext struct { - Post client.Post - Descendants []client.Post - Tags []client.Tag -} - -func New(dir string) (FileWriter, error) { - var fileWriter FileWriter - _, err := os.Stat(dir) - - if os.IsNotExist(err) { - os.Mkdir(dir, os.ModePerm) - } - - absDir, err := filepath.Abs(dir) - - if err != nil { - return fileWriter, err - } - - return FileWriter{ - dir: absDir, - repies: make(map[string]client.Post), - }, nil -} - -func (f FileWriter) Write(post client.Post, threaded bool, templateFile string) error { - if threaded && post.InReplyToId != "" { - f.repies[post.InReplyToId] = post - return nil - } - - var descendants []client.Post - f.getReplies(post.Id, &descendants) - - var file *os.File - var err error - - if len(post.MediaAttachments) == 0 { - name := fmt.Sprintf("%s.md", post.Id) - filename := filepath.Join(f.dir, name) - file, err = os.Create(filename) - } else { - dir := filepath.Join(f.dir, post.Id) - os.Mkdir(dir, os.ModePerm) - - for i := 0; i < len(post.MediaAttachments); i++ { - media := &post.MediaAttachments[i] - if media.Type != "image" { - continue - } - - imageFilename, err := downloadAttachment(dir, media.Id, media.URL) - - if err != nil { - return err - } - - media.Path = imageFilename - } - - filename := filepath.Join(dir, "index.md") - file, err = os.Create(filename) - } - - if err != nil { - return fmt.Errorf("error creating file: %w", err) - } - - defer file.Close() - - tmpl, err := resolveTemplate(templateFile) - context := TemplateContext{ - Post: post, - Descendants: descendants, - Tags: client.TagsForPost(post, descendants), - } - - err = tmpl.Execute(file, context) - - if err != nil { - return fmt.Errorf("error executing template: %w", err) - } - - return nil -} - -func (f FileWriter) getReplies(postId string, replies *[]client.Post) { - if reply, ok := f.repies[postId]; ok { - *replies = append(*replies, reply) - f.getReplies(reply.Id, replies) - } -} - -func downloadAttachment(dir string, id string, url string) (string, error) { - var filename string - - client := &http.Client{} - req, _ := http.NewRequest("GET", url, nil) - req.Header.Set("Accept", "image/*") - res, err := client.Do(req) - - if err != nil { - return filename, err - } - - defer res.Body.Close() - - contentType := res.Header.Get("Content-Type") - extensions, err := mime.ExtensionsByType(contentType) - - if err != nil { - return filename, err - } - - var extension string - urlExtension := filepath.Ext(url) - - for _, i := range extensions { - if i == urlExtension { - extension = i - break - } - } - - if extension == "" { - return filename, fmt.Errorf("could not match extension for media") - } - - filename = fmt.Sprintf("%s%s", id, extension) - file, err := os.Create(filepath.Join(dir, filename)) - - if err != nil { - return filename, err - } - - defer file.Close() - _, err = io.Copy(file, res.Body) - - if err != nil { - return filename, err - } - - return filename, nil -} - -func resolveTemplate(templateFile string) (*template.Template, error) { - converter := md.NewConverter("", true, nil) - - funcs := template.FuncMap{ - "tomd": converter.ConvertString, - } - - if templateFile == "" { - tmpl, err := template.New("post.tmpl").Funcs(funcs).ParseFS(templates, "templates/*.tmpl") - - if err != nil { - return tmpl, err - } - - return tmpl, nil - } - - tmpl, err := template.New(filepath.Base(templateFile)).Funcs(funcs).ParseGlob(templateFile) - - if err != nil { - return tmpl, err - } - - return tmpl, nil -} -======= package files import ( @@ -219,7 +22,7 @@ type FileWriter struct { } type TemplateContext struct { - Post client.Post + Post *client.Post } type PostFile struct { @@ -247,7 +50,7 @@ func New(dir string) (FileWriter, error) { }, nil } -func (f FileWriter) Write(post client.Post, templateFile string) error { +func (f FileWriter) Write(post *client.Post, templateFile string) error { hasMedia := len(post.AllMedia()) > 0 postFile, err := f.createFile(post, hasMedia) @@ -286,7 +89,7 @@ func (f FileWriter) Write(post client.Post, templateFile string) error { return nil } -func (f FileWriter) createFile(post client.Post, shouldBundle bool) (PostFile, error) { +func (f FileWriter) createFile(post *client.Post, shouldBundle bool) (PostFile, error) { var postFile PostFile if shouldBundle { @@ -425,4 +228,3 @@ func resolveTemplate(templateFile string) (*template.Template, error) { return tmpl, nil } ->>>>>>> Stashed changes diff --git a/files/templates/post.tmpl b/files/templates/post.tmpl index 85caa09..9f4365c 100644 --- a/files/templates/post.tmpl +++ b/files/templates/post.tmpl @@ -19,28 +19,13 @@ descendants: {{- end }} --- {{ .Post.Content | tomd }} -<<<<<<< Updated upstream -{{- range .Post.MediaAttachments }} -{{ if eq .Type "image" }} -======= {{ range .Post.MediaAttachments }} {{- if eq .Type "image" }} ->>>>>>> Stashed changes ![{{ .Description }}]({{ .Path }}) {{ end }} {{- end -}} -<<<<<<< Updated upstream -{{- range .Descendants }} -{{ .Content | tomd -}} -{{- range .MediaAttachments }} -{{ if eq .Type "image" }} -![{{ .Description }}]({{ .Path }}) -{{ end }} -{{- end -}} -{{- end -}} -======= {{ range .Post.Descendants }} {{ .Content | tomd }} {{ range .MediaAttachments }} @@ -49,4 +34,3 @@ descendants: {{- end }} {{- end }} {{- end }} ->>>>>>> Stashed changes diff --git a/main.go b/main.go index 844e4e2..4cb39be 100644 --- a/main.go +++ b/main.go @@ -1,4 +1,3 @@ -<<<<<<< Updated upstream package main import ( @@ -28,46 +27,36 @@ func main() { flag.Parse() - c, err := client.New(*user) - - if err != nil { - log.Panicln(fmt.Errorf("error instantiating client: %w", err)) - } - - posts, err := c.Posts(client.PostsFilter{ + c, err := client.New(*user, client.PostsFilter{ ExcludeReplies: *excludeReplies, ExcludeReblogs: *excludeReblogs, Limit: *limit, SinceId: *sinceId, MaxId: *maxId, MinId: *minId, - }) + }, *threaded) if err != nil { log.Panicln(err) } fileWriter, err := files.New(*dist) + posts := c.Posts() + postsCount := len(posts) - if err != nil { - log.Panicln(err) - } - - log.Println(fmt.Sprintf("Fetched %d posts", len(posts))) + log.Println(fmt.Sprintf("Fetched %d posts", postsCount)) for _, post := range posts { - if client.ShouldSkipPost(post) { + if post.ShouldSkip() { continue } - if err := fileWriter.Write(post, *threaded, *templateFile); err != nil { + if err := fileWriter.Write(post, *templateFile); err != nil { log.Panicln("error writing post to file: %w", err) break } } - postsCount := len(posts) - if postsCount > 0 { if *persistFirst != "" { firstPost := posts[0] @@ -102,98 +91,3 @@ func persistId(postId string, path string) error { return nil } -======= -package main - -import ( - "flag" - "fmt" - "log" - "os" - "path/filepath" - - "git.garrido.io/gabriel/mastodon-markdown-archive/client" - "git.garrido.io/gabriel/mastodon-markdown-archive/files" -) - -func main() { - dist := flag.String("dist", "./posts", "Path to directory where files will be written") - user := flag.String("user", "", "URL of User's Mastodon account whose toots will be fetched") - excludeReplies := flag.Bool("exclude-replies", false, "Whether or not exclude replies to other users") - excludeReblogs := flag.Bool("exclude-reblogs", false, "Whether or not to exclude reblogs") - limit := flag.Int("limit", 40, "Maximum number of posts to fetch") - sinceId := flag.String("since-id", "", "Fetch posts greater than this id") - maxId := flag.String("max-id", "", "Fetch posts lesser than this id") - minId := flag.String("min-id", "", "Fetch posts immediately newer than this id") - persistFirst := flag.String("persist-first", "", "Location to persist the post id of the first post returned") - persistLast := flag.String("persist-last", "", "Location to persist the post id of the last post returned") - templateFile := flag.String("template", "", "Template to use for post rendering, if passed") - threaded := flag.Bool("threaded", true, "Thread replies for a post in a single file") - - flag.Parse() - - c, err := client.New(*user, client.PostsFilter{ - ExcludeReplies: *excludeReplies, - ExcludeReblogs: *excludeReblogs, - Limit: *limit, - SinceId: *sinceId, - MaxId: *maxId, - MinId: *minId, - }, *threaded) - - if err != nil { - log.Panicln(err) - } - - fileWriter, err := files.New(*dist) - posts := c.Posts() - postsCount := len(posts) - - log.Println(fmt.Sprintf("Fetched %d posts", postsCount)) - - for _, post := range posts { - if post.ShouldSkip() { - continue - } - - if err := fileWriter.Write(post, *templateFile); err != nil { - log.Panicln("error writing post to file: %w", err) - break - } - } - - if postsCount > 0 { - if *persistFirst != "" { - firstPost := posts[0] - err := persistId(firstPost.Id, *persistFirst) - - if err != nil { - log.Panicln(err) - } - } - - if *persistLast != "" { - lastPost := posts[postsCount-1] - err := persistId(lastPost.Id, *persistLast) - - if err != nil { - log.Panicln(err) - } - } - } -} - -func persistId(postId string, path string) error { - persistPath, err := filepath.Abs(path) - - if err != nil { - return err - } - - if err := os.WriteFile(persistPath, []byte(postId), 0644); err != nil { - return err - } - - return nil -} ->>>>>>> Stashed changes