diff --git a/cmd/main.go b/cmd/main.go index 0f5b8d4..f33074e 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -20,13 +20,14 @@ func main() { server := server.NewServer() uow := repositories.NewUnitOfWork(); + indexRepo := repositories.NewIndexRepository() linkRepo := repositories.NewLinkRepository(*persistence.NewReadContext()) tagRepo := repositories.NewTagRepository() noteRepo := repositories.NewNoteRepository(*persistence.NewReadContext()) tagService := services.NewTagService(tagRepo) linkService := services.NewLinkService(linkRepo) noteService := services.NewNoteService(tagRepo, linkRepo, noteRepo) - idxr := persistence.NewIndexRebuilder() + idxr := services.NewIndexRebuilder(uow, noteRepo, linkRepo, tagRepo, indexRepo ) server.RegisterHandler("vault/init", vault.NewInitializeHandler(idxr)) server.RegisterHandler("note/create", note.NewCreateNoteHandler(uow, tagService, noteRepo)) diff --git a/core/handlers/vault/initialize_handler.go b/core/handlers/vault/initialize_handler.go index e9bccf4..2c345ad 100644 --- a/core/handlers/vault/initialize_handler.go +++ b/core/handlers/vault/initialize_handler.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" + "github.com/KristianJBorgwarth/dendrite.daemon/core/services" "github.com/KristianJBorgwarth/dendrite.daemon/persistence" "github.com/KristianJBorgwarth/dendrite.daemon/persistence/store" ) @@ -15,10 +16,10 @@ type initializeCommand struct { } type InitializeHandler struct { - idxRebuilder persistence.IIndexRebuilder + idxRebuilder services.IIndexRebuilder } -func NewInitializeHandler(idxR persistence.IIndexRebuilder) *InitializeHandler { +func NewInitializeHandler(idxR services.IIndexRebuilder) *InitializeHandler { return &InitializeHandler{idxRebuilder: idxR} } diff --git a/core/services/index_rebuilder.go b/core/services/index_rebuilder.go new file mode 100644 index 0000000..564cf18 --- /dev/null +++ b/core/services/index_rebuilder.go @@ -0,0 +1,173 @@ +package services + +import ( + "context" + "io/fs" + "log/slog" + "path/filepath" + "slices" + "strings" + + filehandling "github.com/KristianJBorgwarth/dendrite.daemon/core/file_handling" + "github.com/KristianJBorgwarth/dendrite.daemon/core/models" + "github.com/KristianJBorgwarth/dendrite.daemon/persistence" + "github.com/KristianJBorgwarth/dendrite.daemon/persistence/repositories" +) + +type IIndexRebuilder interface { + RebuildIndex(ctx context.Context, vaultRoot string) error +} + +type indexRebuilder struct { + uow *repositories.UnitOfWork + noteRepo repositories.INoteRepository + linkRepo repositories.ILinkRepository + tagRepo repositories.ITagRepository + indexRepo repositories.IIndexRepository +} + +func NewIndexRebuilder( + uow *repositories.UnitOfWork, + noteRepo repositories.INoteRepository, + linkRepo repositories.ILinkRepository, + tagRepo repositories.ITagRepository, + indexRepo repositories.IIndexRepository, +) *indexRebuilder { + return &indexRebuilder{ + uow: uow, + noteRepo: noteRepo, + linkRepo: linkRepo, + tagRepo: tagRepo, + indexRepo: indexRepo, + } +} + +func (r *indexRebuilder) RebuildIndex(ctx context.Context, vaultRoot string) error { + files, err := r.readFiles(vaultRoot) + if err != nil { + slog.Debug("Failed to read files from vault", "vaultRoot", vaultRoot, "error", err) + return err + } + + slog.Debug("Successfully read files from vault", "vaultRoot", vaultRoot, "fileCount", len(files)) + + notes, links, tags, noteTags := r.buildDBModels(files) + + dbctx, err := r.uow.Begin() + if err != nil { + return err + } + + if err = r.indexRepo.WipeIndex(ctx, dbctx); err != nil { + r.uow.Rollback() + slog.Debug("Failed to wipe index, rolling back transaction", "error", err) + return err + } + + if err = r.buildIndex(ctx, dbctx, notes, links, tags, noteTags); err != nil { + r.uow.Rollback() + slog.Debug("Failed to build index, rolling back transaction", "error", err) + return err + } + + return nil +} + +func (r *indexRebuilder) buildIndex( + ctx context.Context, + dbctx persistence.IDbContext, + notes []*models.Note, + links []*models.Link, + tags []*models.Tag, + noteTags []*models.NoteTag, +) error { + if err := r.noteRepo.InsertRange(ctx, dbctx, notes); err != nil { + return err + } + + if err := r.linkRepo.InsertRange(ctx, dbctx, links); err != nil { + return err + } + + if err := r.tagRepo.InsertRange(ctx, dbctx, tags); err != nil { + return err + } + + if err := r.tagRepo.InsertNoteTags(ctx, dbctx, noteTags); err != nil { + return err + } + + return nil +} + +func (r *indexRebuilder) readFiles(vault string) ([]*filehandling.File, error) { + var files []*filehandling.File + + err := filepath.WalkDir(vault, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + slog.Debug("Processing file during index rebuild", "path", path) + + if d.IsDir() { + if !r.shouldIndexDirectory(path) { + slog.Debug("Skipping directory during index rebuild", "path", path) + return filepath.SkipDir + } + return nil + } + + if filepath.Ext(path) != ".md" { + slog.Debug("Skipping non-markdown file during index rebuild", "path", path) + return nil + } + + pendingFile, err := filehandling.ReadFile(path) + if err != nil { + return err + } + + files = append(files, pendingFile) + + return nil + }) + if err != nil { + return nil, err + } + + return files, nil +} + +func (r *indexRebuilder) shouldIndexDirectory(path string) bool { + ignoredDirs := []string{".git", ".templates", "temp", "issues"} + for part := range strings.SplitSeq(path, string(filepath.Separator)) { + if slices.Contains(ignoredDirs, part) { + return false + } + } + return true +} + +func (r *indexRebuilder) buildDBModels(files []*filehandling.File) ([]*models.Note, []*models.Link, []*models.Tag, []*models.NoteTag) { + var notes []*models.Note + var links []*models.Link + tagMap := make(map[string]*models.Tag) + var noteTags []*models.NoteTag + + for _, file := range files { + note := models.CreateNote(file.Path, file.Title, file.Slug) + notes = append(notes, note) + for _, t := range models.CreateTags(file.FrontMatter.Tags) { + tagMap[t.Name()] = t + } + noteTags = append(noteTags, models.CreateNoteTags(note.ID(), file.FrontMatter.Tags)...) + links = append(links, models.MapToLinkModel(note.ID(), file.ExtractedLinks)...) + } + + tags := make([]*models.Tag, 0, len(tagMap)) + for _, tag := range tagMap { + tags = append(tags, tag) + } + + return notes, links, tags, noteTags +} diff --git a/core/services/tag_service.go b/core/services/tag_service.go index 5e45caa..14e2c81 100644 --- a/core/services/tag_service.go +++ b/core/services/tag_service.go @@ -58,5 +58,7 @@ func(s *tagService) CreateNoteTags(ctx context.Context, dbCtx persistence.IDbCon return t.Name() }); - return s.tagRepo.InsertNoteTags(ctx, dbCtx, noteID, tagIds) + noteTags := models.CreateNoteTags(noteID, tagIds) + + return s.tagRepo.InsertNoteTags(ctx, dbCtx, noteTags) } diff --git a/dendrite b/dendrite index 30b9cb9..084b912 100755 Binary files a/dendrite and b/dendrite differ diff --git a/persistence/idb_context.go b/persistence/idb_context.go index 13a2225..9dfd1a9 100644 --- a/persistence/idb_context.go +++ b/persistence/idb_context.go @@ -9,4 +9,5 @@ type IDbContext interface { ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row + Prepare(string) (*sql.Stmt, error) } diff --git a/persistence/index_rebuilder.go b/persistence/index_rebuilder.go deleted file mode 100644 index 2bf4b17..0000000 --- a/persistence/index_rebuilder.go +++ /dev/null @@ -1,210 +0,0 @@ -package persistence - -import ( - "context" - "database/sql" - "io/fs" - "log/slog" - "path/filepath" - "slices" - "strings" - - filehandling "github.com/KristianJBorgwarth/dendrite.daemon/core/file_handling" - "github.com/KristianJBorgwarth/dendrite.daemon/core/models" -) - -type IIndexRebuilder interface { - RebuildIndex(ctx context.Context, vaultRoot string) error -} - -type IndexRebuilder struct{} - -func NewIndexRebuilder() *IndexRebuilder { - return &IndexRebuilder{} -} - -func (r *IndexRebuilder) RebuildIndex(ctx context.Context, vaultRoot string) error { - files, err := r.readFiles(vaultRoot) - if err != nil { - slog.Debug("Failed to read files from vault", "vaultRoot", vaultRoot, "error", err) - return err - } - - notes, links, tags, noteTags := r.buildDBModels(files) - - tx, err := GetDBContext().DB.Begin() - if err != nil { - return err - } - - if err = r.wipeIndex(ctx, tx); err != nil { - tx.Rollback() - slog.Debug("Failed to wipe index, rolling back transaction", "error", err) - return err - } - - if err = r.InsertNotes(ctx, tx, notes); err != nil { - tx.Rollback() - return err - } - - if err = r.InsertLinks(ctx, tx, links); err != nil { - tx.Rollback() - return err - } - - if err = r.InsertTags(ctx, tx, tags); err != nil { - tx.Rollback() - return err - } - - if err = r.InsertNoteTags(ctx, tx, noteTags); err != nil { - tx.Rollback() - return err - } - - tx.Commit() - - return nil -} - -func (r *IndexRebuilder) InsertNotes(ctx context.Context, tx *sql.Tx, notes []*models.Note) error { - noteStmt, err := tx.Prepare(`INSERT INTO note (id, title, slug, path, created_at, updated_at) VALUES (?, ?, ?, ?, datetime('now'), datetime('now'))`) - if err != nil { - return err - } - for _, note := range notes { - if _, err = noteStmt.ExecContext(ctx, note.ID(), note.Title(), note.Slug(), note.Path()); err != nil { - return err - } - } - - return nil -} - -func (r *IndexRebuilder) InsertLinks(ctx context.Context, tx *sql.Tx, links []*models.Link) error { - linkStmt, err := tx.Prepare(`INSERT INTO link (id, from_note_id, target_slug, display, raw, line, col) VALUES (?, ?, ?, ?, ?, ?, ?)`) - if err != nil { - return err - } - - for _, link := range links { - if _, err = linkStmt.ExecContext(ctx, link.ID(), link.FromNoteID(), link.TargetSlug(), link.Display(), link.Raw(), link.Line(), link.Col()); err != nil { - return err - } - } - - return nil -} - -func (r *IndexRebuilder) InsertTags(ctx context.Context, tx *sql.Tx, tags []*models.Tag) error { - tagStmst, err := tx.Prepare(`INSERT INTO tag (name) VALUES (?)`) - if err != nil { - return err - } - - for _, tag := range tags { - if _, err = tagStmst.ExecContext(ctx, tag.Name()); err != nil { - return err - } - } - - return nil -} - -func (r *IndexRebuilder) InsertNoteTags(ctx context.Context, tx *sql.Tx, noteTags []*models.NoteTag) error { - noteTagStmt, err := tx.Prepare(`INSERT INTO note_tag (note_id, tag_id) VALUES (?, ?)`) - if err != nil { - return err - } - - for _, noteTag := range noteTags { - if _, err = noteTagStmt.ExecContext(ctx, noteTag.NoteID(), noteTag.TagID()); err != nil { - return err - } - } - - return nil -} - -func (r *IndexRebuilder) readFiles(vault string) ([]*filehandling.File, error) { - var files []*filehandling.File - - err := filepath.WalkDir(vault, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - slog.Debug("Processing file during index rebuild", "path", path) - - if d.IsDir() { - if !r.shouldIndexDirectory(path) { - slog.Debug("Skipping directory during index rebuild", "path", path) - return filepath.SkipDir - } - return nil - } - - if filepath.Ext(path) != ".md" { - slog.Debug("Skipping non-markdown file during index rebuild", "path", path) - return nil - } - - pendingFile, err := filehandling.ReadFile(path) - if err != nil { - return err - } - - files = append(files, pendingFile) - - return nil - }) - if err != nil { - return nil, err - } - - return files, nil -} - -func (r *IndexRebuilder) shouldIndexDirectory(path string) bool { - ignoredDirs := []string{".git", ".templates", "temp", "issues"} - for part := range strings.SplitSeq(path, string(filepath.Separator)) { - if slices.Contains(ignoredDirs, part) { - return false - } - } - return true -} - -func (r *IndexRebuilder) buildDBModels(files []*filehandling.File) ([]*models.Note, []*models.Link, []*models.Tag, []*models.NoteTag) { - var notes []*models.Note - var links []*models.Link - tagMap := make(map[string]*models.Tag) - var noteTags []*models.NoteTag - - for _, file := range files { - note := models.CreateNote(file.Path, file.Title, file.Slug) - notes = append(notes, note) - for _, t := range models.CreateTags(file.FrontMatter.Tags) { - tagMap[t.Name()] = t - } - noteTags = append(noteTags, models.CreateNoteTags(note.ID(), file.FrontMatter.Tags)...) - links = append(links, models.MapToLinkModel(note.ID(), file.ExtractedLinks)...) - } - - tags := make([]*models.Tag, 0, len(tagMap)) - for _, tag := range tagMap { - tags = append(tags, tag) - } - - return notes, links, tags, noteTags -} - -func (r *IndexRebuilder) wipeIndex(ctx context.Context, tx *sql.Tx) error { - cmd := `DELETE FROM note; - DELETE FROM tag; - DELETE FROM note_tag; - DELETE FROM link;` - - _, err := tx.ExecContext(ctx, cmd) - return err -} diff --git a/persistence/repositories/index_repository.go b/persistence/repositories/index_repository.go new file mode 100644 index 0000000..edf32f4 --- /dev/null +++ b/persistence/repositories/index_repository.go @@ -0,0 +1,28 @@ +package repositories + +import ( + "context" + + "github.com/KristianJBorgwarth/dendrite.daemon/persistence" +) + +type IIndexRepository interface { + WipeIndex(ctx context.Context, dbContext persistence.IDbContext) error +} + +type indexRepository struct{ +} + +func NewIndexRepository() IIndexRepository { + return &indexRepository{} +} + +func (r *indexRepository) WipeIndex(ctx context.Context, dbContext persistence.IDbContext) error { + cmd := `DELETE FROM note; + DELETE FROM tag; + DELETE FROM note_tag; + DELETE FROM link;` + + _, err := dbContext.ExecContext(ctx, cmd) + return err +} diff --git a/persistence/repositories/link_repository.go b/persistence/repositories/link_repository.go index 957b921..2c63136 100644 --- a/persistence/repositories/link_repository.go +++ b/persistence/repositories/link_repository.go @@ -10,6 +10,7 @@ import ( type ILinkRepository interface { Insert(ctx context.Context, dbContext persistence.IDbContext, links []*models.Link) error + InsertRange(ctx context.Context, dbContext persistence.IDbContext, links []*models.Link) error GetByNoteID(ctx context.Context, dbContext persistence.IDbContext, fromNoteID string) ([]*models.Link, error) GetBySlug(ctx context.Context, dbContext persistence.IDbContext, targetSlug string) ([]*models.Link, error) Search(ctx context.Context, query string) ([]*models.Link, error) @@ -43,6 +44,25 @@ func (r *linkRepository) Insert(ctx context.Context, dbContext persistence.IDbCo return err } +func (r *linkRepository) InsertRange(ctx context.Context, dbContext persistence.IDbContext, links []*models.Link) error { + if len(links) == 0 { + return nil + } + + linkStatement, err := dbContext.Prepare(`INSERT OR IGNORE INTO link (id, from_note_id, target_slug, raw, display, line, col) VALUES (?, ?, ?, ?, ?, ?, ?)`) + if err != nil { + return err + } + + for _, link := range links { + if _, err := linkStatement.ExecContext(ctx, link.ID(), link.FromNoteID(), link.TargetSlug(), link.Raw(), link.Display(), link.Line(), link.Col()); err != nil { + return err + } + } + + return nil +} + func (r *linkRepository) GetByNoteID(ctx context.Context, dbContext persistence.IDbContext, fromNoteID string) ([]*models.Link, error) { rows, err := dbContext.QueryContext(ctx, "SELECT id, from_note_id, target_slug, raw, display, line, col FROM link WHERE from_note_id = ?", fromNoteID) if err != nil { diff --git a/persistence/repositories/note_repository.go b/persistence/repositories/note_repository.go index 5f8741f..d73a787 100644 --- a/persistence/repositories/note_repository.go +++ b/persistence/repositories/note_repository.go @@ -11,11 +11,12 @@ import ( type INoteRepository interface { Insert(ctx context.Context, dbContext persistence.IDbContext, note *models.Note) error + InsertRange(ctx context.Context, dbContext persistence.IDbContext, note []*models.Note) error Update(ctx context.Context, dbContext persistence.IDbContext, noteID, path, title, slug string) error GetBySlug(ctx context.Context, slug string) (*models.Note, error) } -type noteRepository struct{ +type noteRepository struct { readDBContext persistence.ReadContext } @@ -35,6 +36,24 @@ func (r *noteRepository) Insert(ctx context.Context, dbContext persistence.IDbCo return err } +func (r *noteRepository) InsertRange(ctx context.Context, dbContext persistence.IDbContext, notes []*models.Note) error { + if len(notes) == 0 { + return nil + } + noteStatement, err := dbContext.Prepare(` + INSERT INTO note (id, title, path, slug, created_at, updated_at) + VALUES (?, ?, ?, ?, datetime('now'), datetime('now'))`) + if err != nil { + return err + } + for _, note := range notes { + if _, err := noteStatement.ExecContext(ctx, note.ID(), note.Title(), note.Path(), note.Slug()); err != nil { + return err + } + } + return nil +} + func (r *noteRepository) Update(ctx context.Context, dbContext persistence.IDbContext, noteID, path, title, slug string) error { query := ` UPDATE note diff --git a/persistence/repositories/tag_repository.go b/persistence/repositories/tag_repository.go index 79d19d4..cd0c6f3 100644 --- a/persistence/repositories/tag_repository.go +++ b/persistence/repositories/tag_repository.go @@ -10,7 +10,8 @@ import ( type ITagRepository interface { Insert(ctx context.Context, dbCtx persistence.IDbContext, tags []*models.Tag) error - InsertNoteTags(ctx context.Context, dbCtx persistence.IDbContext, noteID string, tagIDs []string) error + InsertRange(ctx context.Context, dbCtx persistence.IDbContext, tags []*models.Tag) error + InsertNoteTags(ctx context.Context, dbCtx persistence.IDbContext, noteTags []*models.NoteTag) error GetByNames(ctx context.Context, dbCtx persistence.IDbContext, names []string) ([]*models.Tag, error) DeleteNoteTags(ctx context.Context, dbCtx persistence.IDbContext, noteID string) error } @@ -40,26 +41,37 @@ func (r *tagRepository) Insert(ctx context.Context, dbCtx persistence.IDbContext return err } -func (r *tagRepository) InsertNoteTags(ctx context.Context, dbCtx persistence.IDbContext, noteID string, tags []string) error { +func (r *tagRepository) InsertRange(ctx context.Context, dbCtx persistence.IDbContext, tags []*models.Tag) error { if len(tags) == 0 { return nil } - placeholders := make([]string, 0, len(tags)) - args := make([]any, 0, len(tags)) - - for _, tagID := range tags { - placeholders = append(placeholders, "(?, ?)") - args = append(args, noteID, tagID) - } - - query := "INSERT OR IGNORE INTO note_tag(note_id, tag_id) VALUES " + strings.Join(placeholders, ",") - - _, err := dbCtx.ExecContext(ctx, query, args...) + tagStatement, err := dbCtx.Prepare(`INSERT OR IGNORE INTO tag (name) VALUES (?)`) if err != nil { return err } + for _, tag := range tags { + if _, err = tagStatement.ExecContext(ctx, tag.Name()); err != nil { + return err + } + } + + return nil +} + +func (r *tagRepository) InsertNoteTags(ctx context.Context, dbCtx persistence.IDbContext, noteTags []*models.NoteTag) error { + noteTagStmt, err := dbCtx.Prepare(`INSERT INTO note_tag (note_id, tag_id) VALUES (?, ?)`) + if err != nil { + return err + } + + for _, noteTag := range noteTags { + if _, err = noteTagStmt.ExecContext(ctx, noteTag.NoteID(), noteTag.TagID()); err != nil { + return err + } + } + return nil }