Merge pull request #28 from KristianJBorgwarth/feat/setup-index

feat(index): wipe and rebuild index

cmd/main.go

@@ -26,8 +26,9 @@ func main() {
 	tagService := services.NewTagService(tagRepo)
 	linkService := services.NewLinkService(linkRepo)
 	noteService := services.NewNoteService(tagRepo, linkRepo, noteRepo)
+	idxr := persistence.NewIndexRebuilder()
 
-	server.RegisterHandler("vault/init", vault.NewInitializeHandler())
+	server.RegisterHandler("vault/init", vault.NewInitializeHandler(idxr))
 	server.RegisterHandler("note/create", note.NewCreateNoteHandler(uow, tagService, noteRepo))
 	server.RegisterHandler("note/save", note.NewSaveNoteHandler(uow, noteRepo, tagService, noteService, linkService))
 	server.RegisterHandler("note/goto", note.NewGotoNoteHandler(noteRepo))

core/file_handling/file.go

@@ -38,6 +38,12 @@ func ReadFile(path string) (*File, error) {
 		return nil, err
 	}
 
+	if fm == nil {
+		fm = &FrontMatter{
+			Title: strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)),
+		}
+	}
+
 	return &File{
 		Path:        path,
 		Title:       fm.Title,

core/file_handling/frontmatter_parser.go

@@ -2,7 +2,6 @@ package filehandling
 
 import (
 	"bytes"
-	"errors"
 
 	"gopkg.in/yaml.v3"
 )
@@ -12,6 +11,7 @@ type FrontMatter struct {
 	Tags    []string `yaml:"tags"`
 	Created string   `yaml:"created"`
 	Updated string   `yaml:"updated"`
+	Date		string   `yaml:"date"`
 	Author  string   `yaml:"author"`
 }
 
@@ -20,7 +20,7 @@ func ParseFrontMatter(file []byte) (*FrontMatter, error) {
 	if bytes.HasPrefix(content, []byte("---")) {
 		content = bytes.TrimSpace(content[3:])
 	} else {
-		return nil, errors.New("missing front matter delimiter")
+		return nil, nil
 	}
 
 	if idx := bytes.Index(content, []byte("---")); idx != -1 {

core/handlers/vault/initialize_handler.go

@@ -14,10 +14,12 @@ type initializeCommand struct {
 	TemplateDirectory string `json:"templateDirectory"`
 }
 
-type InitializeHandler struct{}
+type InitializeHandler struct {
+	idxRebuilder persistence.IIndexRebuilder
+}
 
-func NewInitializeHandler() *InitializeHandler {
+func NewInitializeHandler(idxR persistence.IIndexRebuilder) *InitializeHandler {
-	return &InitializeHandler{}
+	return &InitializeHandler{idxRebuilder: idxR}
 }
 
 func (h InitializeHandler) Handle(ctx context.Context, raw json.RawMessage) (any, error) {
@@ -34,5 +36,9 @@ func (h InitializeHandler) Handle(ctx context.Context, raw json.RawMessage) (any
 		return nil, err
 	}
 
+	if err = h.idxRebuilder.RebuildIndex(ctx, cmd.VaultPath); err != nil {
+		return nil, err
+	}
+
 	return nil, nil
 }

persistence/index_rebuilder.go

@@ -0,0 +1,210 @@
+package persistence
+
+import (
+	"context"
+	"database/sql"
+	"io/fs"
+	"log/slog"
+	"path/filepath"
+	"slices"
+	"strings"
+
+	filehandling "github.com/KristianJBorgwarth/dendrite.daemon/core/file_handling"
+	"github.com/KristianJBorgwarth/dendrite.daemon/core/models"
+)
+
+type IIndexRebuilder interface {
+	RebuildIndex(ctx context.Context, vaultRoot string) error
+}
+
+type IndexRebuilder struct{}
+
+func NewIndexRebuilder() *IndexRebuilder {
+	return &IndexRebuilder{}
+}
+
+func (r *IndexRebuilder) RebuildIndex(ctx context.Context, vaultRoot string) error {
+	files, err := r.readFiles(vaultRoot)
+	if err != nil {
+		slog.Debug("Failed to read files from vault", "vaultRoot", vaultRoot, "error", err)
+		return err
+	}
+
+	notes, links, tags, noteTags := r.buildDBModels(files)
+
+	tx, err := GetDBContext().DB.Begin()
+	if err != nil {
+		return err
+	}
+
+	if err = r.wipeIndex(ctx, tx); err != nil {
+		tx.Rollback()
+		slog.Debug("Failed to wipe index, rolling back transaction", "error", err)
+		return err
+	}
+
+	if err = r.InsertNotes(ctx, tx, notes); err != nil {
+		tx.Rollback()
+		return err
+	}
+
+	if err = r.InsertLinks(ctx, tx, links); err != nil {
+		tx.Rollback()
+		return err
+	}
+
+	if err = r.InsertTags(ctx, tx, tags); err != nil {
+		tx.Rollback()
+		return err
+	}
+
+	if err = r.InsertNoteTags(ctx, tx, noteTags); err != nil {
+		tx.Rollback()
+		return err
+	}
+
+	tx.Commit()
+
+	return nil
+}
+
+func (r *IndexRebuilder) InsertNotes(ctx context.Context, tx *sql.Tx, notes []*models.Note) error {
+	noteStmt, err := tx.Prepare(`INSERT INTO note (id, title, slug, path, created_at, updated_at) VALUES (?, ?, ?, ?, datetime('now'), datetime('now'))`)
+	if err != nil {
+		return err
+	}
+	for _, note := range notes {
+		if _, err = noteStmt.ExecContext(ctx, note.ID(), note.Title(), note.Slug(), note.Path()); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (r *IndexRebuilder) InsertLinks(ctx context.Context, tx *sql.Tx, links []*models.Link) error {
+	linkStmt, err := tx.Prepare(`INSERT INTO link (id, from_note_id, target_slug, display, raw, line, col) VALUES (?, ?, ?, ?, ?, ?, ?)`)
+	if err != nil {
+		return err
+	}
+
+	for _, link := range links {
+		if _, err = linkStmt.ExecContext(ctx, link.ID(), link.FromNoteID(), link.TargetSlug(), link.Display(), link.Raw(), link.Line(), link.Col()); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (r *IndexRebuilder) InsertTags(ctx context.Context, tx *sql.Tx, tags []*models.Tag) error {
+	tagStmst, err := tx.Prepare(`INSERT INTO tag (name) VALUES (?)`)
+	if err != nil {
+		return err
+	}
+
+	for _, tag := range tags {
+		if _, err = tagStmst.ExecContext(ctx, tag.Name()); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (r *IndexRebuilder) InsertNoteTags(ctx context.Context, tx *sql.Tx, noteTags []*models.NoteTag) error {
+	noteTagStmt, err := tx.Prepare(`INSERT INTO note_tag (note_id, tag_id) VALUES (?, ?)`)
+	if err != nil {
+		return err
+	}
+
+	for _, noteTag := range noteTags {
+		if _, err = noteTagStmt.ExecContext(ctx, noteTag.NoteID(), noteTag.TagID()); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (r *IndexRebuilder) readFiles(vault string) ([]*filehandling.File, error) {
+	var files []*filehandling.File
+
+	err := filepath.WalkDir(vault, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		slog.Debug("Processing file during index rebuild", "path", path)
+
+		if d.IsDir() {
+			if !r.shouldIndexDirectory(path) {
+				slog.Debug("Skipping directory during index rebuild", "path", path)
+				return filepath.SkipDir
+			}
+			return nil
+		}
+
+		if filepath.Ext(path) != ".md" {
+			slog.Debug("Skipping non-markdown file during index rebuild", "path", path)
+			return nil
+		}
+
+		pendingFile, err := filehandling.ReadFile(path)
+		if err != nil {
+			return err
+		}
+
+		files = append(files, pendingFile)
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return files, nil
+}
+
+func (r *IndexRebuilder) shouldIndexDirectory(path string) bool {
+	ignoredDirs := []string{".git", ".templates", "temp", "issues"}
+	for part := range strings.SplitSeq(path, string(filepath.Separator)) {
+		if slices.Contains(ignoredDirs, part) {
+			return false
+		}
+	}
+	return true
+}
+
+func (r *IndexRebuilder) buildDBModels(files []*filehandling.File) ([]*models.Note, []*models.Link, []*models.Tag, []*models.NoteTag) {
+	var notes []*models.Note
+	var links []*models.Link
+	tagMap := make(map[string]*models.Tag)
+	var noteTags []*models.NoteTag
+
+	for _, file := range files {
+		note := models.CreateNote(file.Path, file.Title, file.Slug)
+		notes = append(notes, note)
+		for _, t := range models.CreateTags(file.FrontMatter.Tags) {
+			tagMap[t.Name()] = t
+		}
+		noteTags = append(noteTags, models.CreateNoteTags(note.ID(), file.FrontMatter.Tags)...)
+		links = append(links, models.MapToLinkModel(note.ID(), file.ExtractedLinks)...)
+	}
+
+	tags := make([]*models.Tag, 0, len(tagMap))
+	for _, tag := range tagMap {
+		tags = append(tags, tag)
+	}
+
+	return notes, links, tags, noteTags
+}
+
+func (r *IndexRebuilder) wipeIndex(ctx context.Context, tx *sql.Tx) error {
+	cmd := `DELETE FROM note; 
+	DELETE FROM tag;
+	DELETE FROM note_tag;
+	DELETE FROM link;`
+
+	_, err := tx.ExecContext(ctx, cmd)
+	return err
+}

persistence/services/doc.go

@@ -1,2 +0,0 @@
-// Package peristenceServices provides services for data persistence, including database interactions and storage management.
-package services

persistence/services/index_rebuilder.go

@@ -1,149 +0,0 @@
-package services
-
-import (
-	"database/sql"
-	"io/fs"
-	"path/filepath"
-
-	filehandling "github.com/KristianJBorgwarth/dendrite.daemon/core/file_handling"
-	"github.com/KristianJBorgwarth/dendrite.daemon/core/models"
-)
-
-type IIndexRebuilder interface {
-	RebuildIndex(vaultRoot string) error
-}
-
-type IndexRebuilder struct {
-	db *sql.DB
-}
-
-func (r *IndexRebuilder) RebuildIndex(vaultRoot string) error {
-	files, err := r.ReadFiles(vaultRoot)
-	if err != nil {
-		return err
-	}
-
-	notes, links, tags, noteTags := r.buildDBModels(files)
-
-	tx, err := r.db.Begin()
-	if err != nil {
-		return err
-	}
-
-	if err = r.wipeIndex(tx); err != nil {
-		tx.Rollback()
-		return err
-	}
-
-	if err = r.InsertNotes(tx, notes); err != nil {
-		tx.Rollback()
-		return err
-	}
-	
-	if err = r.InsertLinks(tx, links); err != nil {
-		tx.Rollback()
-		return err
-	}
-
-	if err = r.InsertTags(tx, tags); err != nil {
-		tx.Rollback()
-		return err
-	}
-
-	if err = r.InsertNoteTags(tx, noteTags); err != nil {
-		tx.Rollback()
-		return err
-	}
-
-	tx.Commit()
-
-	return nil
-}
-
-
-func (r *IndexRebuilder) InsertNotes(tx *sql.Tx, notes []*models.Note) error {
-	noteStmt, err := tx.Prepare(`INSERT INTO note (id, title, slug, path) VALUES (?, ?, ?, ?)`)
-	if err != nil {
-		return err
-	}
-	for _, note := range notes {
-		if _, err = noteStmt.Exec(note.ID(), note.Title(), note.Slug(), note.Path()); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (r *IndexRebuilder) InsertLinks(tx *sql.Tx, links []*models.Link) error {
-	panic("not implemented")
-}
-
-func (r *IndexRebuilder) InsertTags(tx *sql.Tx, tags []*models.Tag) error {
-	panic("not implemented")
-}
-
-func (r *IndexRebuilder) InsertNoteTags(tx *sql.Tx, noteTags []*models.NoteTag) error {
-	panic("not implemented")
-}
-
-func (r *IndexRebuilder) ReadFiles(vault string) ([]*filehandling.File, error) {
-	var files []*filehandling.File
-
-	err := filepath.WalkDir(vault, func(path string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-
-		if d.IsDir() || filepath.Ext(path) != ".md" {
-			return nil
-		}
-		pendingFile, err := filehandling.ReadFile(path)
-		if err != nil {
-			return err
-		}
-
-		files = append(files, pendingFile)
-
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return files, nil
-}
-
-func (r *IndexRebuilder) buildDBModels(files []*filehandling.File) ([]*models.Note, []*models.Link, []*models.Tag, []*models.NoteTag) {
-	var notes []*models.Note
-	var links []*models.Link
-	tagMap := make(map[string]*models.Tag)
-	var noteTags []*models.NoteTag
-
-	for _, file := range files {
-		note := models.CreateNote(file.Path, file.Title, file.Slug)
-		notes = append(notes, note)
-		for _, t := range models.CreateTags(file.FrontMatter.Tags) {
-			tagMap[t.Name()] = t
-		}
-		noteTags = append(noteTags, models.CreateNoteTags(note.ID(), file.FrontMatter.Tags)...)
-		links = append(links, models.MapToLinkModel(note.ID(), file.ExtractedLinks)...)
-	}
-
-	tags := make([]*models.Tag, 0, len(tagMap))
-	for _, tag := range tagMap {
-		tags = append(tags, tag)
-	}
-
-	return notes, links, tags, noteTags
-}
-
-func (r *IndexRebuilder) wipeIndex(tx *sql.Tx) error {
-	cmd := `DELETE FROM notes; 
-	DELETE FROM tags;
-	DELETE FROM note_tags;
-	DELETE FROM link;`
-
-	_, err := tx.Exec(cmd)
-	return err
-}

test/test_unit/frontmatter_test.go

@@ -22,12 +22,13 @@ This is the content of the note.`
 	assert.Equal(t, []string{"test", "note"}, result.Tags)
 }
 
-func TestParseTags_MissingDelimiter_ReturnsError(t *testing.T) {
+func TestParseTags_MissingDelimiter_ReturnsNil(t *testing.T) {
 	input := `title: My Note
 tags: ["test", "note"]
 This is the content of the note.`
 
-	_, err := filehandling.ParseFrontMatter([]byte(input))
+	val, err := filehandling.ParseFrontMatter([]byte(input))
 
-	assert.ErrorContains(t, err, "missing front matter delimiter")
+	assert.NoError(t, err)
+	assert.Nil(t, val)
 }