kristianjbf/dendrite.daemon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore(idx_rebuilder): mv to core and use repos (#29)

Browse source 
* ref(index_rebuilder): mv to core

* ref(idx_rebuilder): moved db acc code to repos

* feat(idx_rebuilder): mv idx rebuilder to core and implemented build
This commit is contained in:
Kristian 2026-04-18 15:57:30 +02:00 • committed by GitHub
parent 8556e07964
commit 478f309108
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 275 additions and 228 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
cmd/main.go
View file

@ -20,13 +20,14 @@ func main() {
server := server.NewServer()
uow := repositories.NewUnitOfWork();
indexRepo := repositories.NewIndexRepository()
linkRepo := repositories.NewLinkRepository(*persistence.NewReadContext())
tagRepo := repositories.NewTagRepository()
noteRepo := repositories.NewNoteRepository(*persistence.NewReadContext())
tagService := services.NewTagService(tagRepo)
linkService := services.NewLinkService(linkRepo)
noteService := services.NewNoteService(tagRepo, linkRepo, noteRepo)
idxr := persistence.NewIndexRebuilder()
idxr := services.NewIndexRebuilder(uow, noteRepo, linkRepo, tagRepo, indexRepo )
server.RegisterHandler("vault/init", vault.NewInitializeHandler(idxr))
server.RegisterHandler("note/create", note.NewCreateNoteHandler(uow, tagService, noteRepo))

5
core/handlers/vault/initialize_handler.go
View file

@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"github.com/KristianJBorgwarth/dendrite.daemon/core/services"
"github.com/KristianJBorgwarth/dendrite.daemon/persistence"
"github.com/KristianJBorgwarth/dendrite.daemon/persistence/store"
)
@ -15,10 +16,10 @@ type initializeCommand struct {
}
type InitializeHandler struct {
idxRebuilder persistence.IIndexRebuilder
idxRebuilder services.IIndexRebuilder
}
func NewInitializeHandler(idxR persistence.IIndexRebuilder) *InitializeHandler {
func NewInitializeHandler(idxR services.IIndexRebuilder) *InitializeHandler {
return &InitializeHandler{idxRebuilder: idxR}
}

173
core/services/index_rebuilder.go Normal file
View file

@ -0,0 +1,173 @@
package services
import (
"context"
"io/fs"
"log/slog"
"path/filepath"
"slices"
"strings"
filehandling "github.com/KristianJBorgwarth/dendrite.daemon/core/file_handling"
"github.com/KristianJBorgwarth/dendrite.daemon/core/models"
"github.com/KristianJBorgwarth/dendrite.daemon/persistence"
"github.com/KristianJBorgwarth/dendrite.daemon/persistence/repositories"
)
type IIndexRebuilder interface {
RebuildIndex(ctx context.Context, vaultRoot string) error
}
type indexRebuilder struct {
uow *repositories.UnitOfWork
noteRepo repositories.INoteRepository
linkRepo repositories.ILinkRepository
tagRepo repositories.ITagRepository
indexRepo repositories.IIndexRepository
}
func NewIndexRebuilder(
uow *repositories.UnitOfWork,
noteRepo repositories.INoteRepository,
linkRepo repositories.ILinkRepository,
tagRepo repositories.ITagRepository,
indexRepo repositories.IIndexRepository,
) *indexRebuilder {
return &indexRebuilder{
uow: uow,
noteRepo: noteRepo,
linkRepo: linkRepo,
tagRepo: tagRepo,
indexRepo: indexRepo,
}
}
func (r *indexRebuilder) RebuildIndex(ctx context.Context, vaultRoot string) error {
files, err := r.readFiles(vaultRoot)
if err != nil {
slog.Debug("Failed to read files from vault", "vaultRoot", vaultRoot, "error", err)
return err
}
slog.Debug("Successfully read files from vault", "vaultRoot", vaultRoot, "fileCount", len(files))
notes, links, tags, noteTags := r.buildDBModels(files)
dbctx, err := r.uow.Begin()
if err != nil {
return err
}
if err = r.indexRepo.WipeIndex(ctx, dbctx); err != nil {
r.uow.Rollback()
slog.Debug("Failed to wipe index, rolling back transaction", "error", err)
return err
}
if err = r.buildIndex(ctx, dbctx, notes, links, tags, noteTags); err != nil {
r.uow.Rollback()
slog.Debug("Failed to build index, rolling back transaction", "error", err)
return err
}
return nil
}
func (r *indexRebuilder) buildIndex(
ctx context.Context,
dbctx persistence.IDbContext,
notes []*models.Note,
links []*models.Link,
tags []*models.Tag,
noteTags []*models.NoteTag,
) error {
if err := r.noteRepo.InsertRange(ctx, dbctx, notes); err != nil {
return err
}
if err := r.linkRepo.InsertRange(ctx, dbctx, links); err != nil {
return err
}
if err := r.tagRepo.InsertRange(ctx, dbctx, tags); err != nil {
return err
}
if err := r.tagRepo.InsertNoteTags(ctx, dbctx, noteTags); err != nil {
return err
}
return nil
}
func (r *indexRebuilder) readFiles(vault string) ([]*filehandling.File, error) {
var files []*filehandling.File
err := filepath.WalkDir(vault, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
slog.Debug("Processing file during index rebuild", "path", path)
if d.IsDir() {
if !r.shouldIndexDirectory(path) {
slog.Debug("Skipping directory during index rebuild", "path", path)
return filepath.SkipDir
}
return nil
}
if filepath.Ext(path) != ".md" {
slog.Debug("Skipping non-markdown file during index rebuild", "path", path)
return nil
}
pendingFile, err := filehandling.ReadFile(path)
if err != nil {
return err
}
files = append(files, pendingFile)
return nil
})
if err != nil {
return nil, err
}
return files, nil
}
func (r *indexRebuilder) shouldIndexDirectory(path string) bool {
ignoredDirs := []string{".git", ".templates", "temp", "issues"}
for part := range strings.SplitSeq(path, string(filepath.Separator)) {
if slices.Contains(ignoredDirs, part) {
return false
}
}
return true
}
func (r *indexRebuilder) buildDBModels(files []*filehandling.File) ([]*models.Note, []*models.Link, []*models.Tag, []*models.NoteTag) {
var notes []*models.Note
var links []*models.Link
tagMap := make(map[string]*models.Tag)
var noteTags []*models.NoteTag
for _, file := range files {
note := models.CreateNote(file.Path, file.Title, file.Slug)
notes = append(notes, note)
for _, t := range models.CreateTags(file.FrontMatter.Tags) {
tagMap[t.Name()] = t
}
noteTags = append(noteTags, models.CreateNoteTags(note.ID(), file.FrontMatter.Tags)...)
links = append(links, models.MapToLinkModel(note.ID(), file.ExtractedLinks)...)
}
tags := make([]*models.Tag, 0, len(tagMap))
for _, tag := range tagMap {
tags = append(tags, tag)
}
return notes, links, tags, noteTags
}

4
core/services/tag_service.go
View file

@ -58,5 +58,7 @@ func(s *tagService) CreateNoteTags(ctx context.Context, dbCtx persistence.IDbCon
return t.Name()
});
return s.tagRepo.InsertNoteTags(ctx, dbCtx, noteID, tagIds)
noteTags := models.CreateNoteTags(noteID, tagIds)
return s.tagRepo.InsertNoteTags(ctx, dbCtx, noteTags)
}

BIN
dendrite
View file

Binary file not shown.

1
persistence/idb_context.go
View file

@ -9,4 +9,5 @@ type IDbContext interface {
ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
Prepare(string) (*sql.Stmt, error)
}

210
persistence/index_rebuilder.go
View file

@ -1,210 +0,0 @@
package persistence
import (
"context"
"database/sql"
"io/fs"
"log/slog"
"path/filepath"
"slices"
"strings"
filehandling "github.com/KristianJBorgwarth/dendrite.daemon/core/file_handling"
"github.com/KristianJBorgwarth/dendrite.daemon/core/models"
)
type IIndexRebuilder interface {
RebuildIndex(ctx context.Context, vaultRoot string) error
}
type IndexRebuilder struct{}
func NewIndexRebuilder() *IndexRebuilder {
return &IndexRebuilder{}
}
func (r *IndexRebuilder) RebuildIndex(ctx context.Context, vaultRoot string) error {
files, err := r.readFiles(vaultRoot)
if err != nil {
slog.Debug("Failed to read files from vault", "vaultRoot", vaultRoot, "error", err)
return err
}
notes, links, tags, noteTags := r.buildDBModels(files)
tx, err := GetDBContext().DB.Begin()
if err != nil {
return err
}
if err = r.wipeIndex(ctx, tx); err != nil {
tx.Rollback()
slog.Debug("Failed to wipe index, rolling back transaction", "error", err)
return err
}
if err = r.InsertNotes(ctx, tx, notes); err != nil {
tx.Rollback()
return err
}
if err = r.InsertLinks(ctx, tx, links); err != nil {
tx.Rollback()
return err
}
if err = r.InsertTags(ctx, tx, tags); err != nil {
tx.Rollback()
return err
}
if err = r.InsertNoteTags(ctx, tx, noteTags); err != nil {
tx.Rollback()
return err
}
tx.Commit()
return nil
}
func (r *IndexRebuilder) InsertNotes(ctx context.Context, tx *sql.Tx, notes []*models.Note) error {
noteStmt, err := tx.Prepare(`INSERT INTO note (id, title, slug, path, created_at, updated_at) VALUES (?, ?, ?, ?, datetime('now'), datetime('now'))`)
if err != nil {
return err
}
for _, note := range notes {
if _, err = noteStmt.ExecContext(ctx, note.ID(), note.Title(), note.Slug(), note.Path()); err != nil {
return err
}
}
return nil
}
func (r *IndexRebuilder) InsertLinks(ctx context.Context, tx *sql.Tx, links []*models.Link) error {
linkStmt, err := tx.Prepare(`INSERT INTO link (id, from_note_id, target_slug, display, raw, line, col) VALUES (?, ?, ?, ?, ?, ?, ?)`)
if err != nil {
return err
}
for _, link := range links {
if _, err = linkStmt.ExecContext(ctx, link.ID(), link.FromNoteID(), link.TargetSlug(), link.Display(), link.Raw(), link.Line(), link.Col()); err != nil {
return err
}
}
return nil
}
func (r *IndexRebuilder) InsertTags(ctx context.Context, tx *sql.Tx, tags []*models.Tag) error {
tagStmst, err := tx.Prepare(`INSERT INTO tag (name) VALUES (?)`)
if err != nil {
return err
}
for _, tag := range tags {
if _, err = tagStmst.ExecContext(ctx, tag.Name()); err != nil {
return err
}
}
return nil
}
func (r *IndexRebuilder) InsertNoteTags(ctx context.Context, tx *sql.Tx, noteTags []*models.NoteTag) error {
noteTagStmt, err := tx.Prepare(`INSERT INTO note_tag (note_id, tag_id) VALUES (?, ?)`)
if err != nil {
return err
}
for _, noteTag := range noteTags {
if _, err = noteTagStmt.ExecContext(ctx, noteTag.NoteID(), noteTag.TagID()); err != nil {
return err
}
}
return nil
}
func (r *IndexRebuilder) readFiles(vault string) ([]*filehandling.File, error) {
var files []*filehandling.File
err := filepath.WalkDir(vault, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
slog.Debug("Processing file during index rebuild", "path", path)
if d.IsDir() {
if !r.shouldIndexDirectory(path) {
slog.Debug("Skipping directory during index rebuild", "path", path)
return filepath.SkipDir
}
return nil
}
if filepath.Ext(path) != ".md" {
slog.Debug("Skipping non-markdown file during index rebuild", "path", path)
return nil
}
pendingFile, err := filehandling.ReadFile(path)
if err != nil {
return err
}
files = append(files, pendingFile)
return nil
})
if err != nil {
return nil, err
}
return files, nil
}
func (r *IndexRebuilder) shouldIndexDirectory(path string) bool {
ignoredDirs := []string{".git", ".templates", "temp", "issues"}
for part := range strings.SplitSeq(path, string(filepath.Separator)) {
if slices.Contains(ignoredDirs, part) {
return false
}
}
return true
}
func (r *IndexRebuilder) buildDBModels(files []*filehandling.File) ([]*models.Note, []*models.Link, []*models.Tag, []*models.NoteTag) {
var notes []*models.Note
var links []*models.Link
tagMap := make(map[string]*models.Tag)
var noteTags []*models.NoteTag
for _, file := range files {
note := models.CreateNote(file.Path, file.Title, file.Slug)
notes = append(notes, note)
for _, t := range models.CreateTags(file.FrontMatter.Tags) {
tagMap[t.Name()] = t
}
noteTags = append(noteTags, models.CreateNoteTags(note.ID(), file.FrontMatter.Tags)...)
links = append(links, models.MapToLinkModel(note.ID(), file.ExtractedLinks)...)
}
tags := make([]*models.Tag, 0, len(tagMap))
for _, tag := range tagMap {
tags = append(tags, tag)
}
return notes, links, tags, noteTags
}
func (r *IndexRebuilder) wipeIndex(ctx context.Context, tx *sql.Tx) error {
cmd := `DELETE FROM note;
DELETE FROM tag;
DELETE FROM note_tag;
DELETE FROM link;`
_, err := tx.ExecContext(ctx, cmd)
return err
}

28
persistence/repositories/index_repository.go Normal file
View file

@ -0,0 +1,28 @@
package repositories
import (
"context"
"github.com/KristianJBorgwarth/dendrite.daemon/persistence"
)
type IIndexRepository interface {
WipeIndex(ctx context.Context, dbContext persistence.IDbContext) error
}
type indexRepository struct{
}
func NewIndexRepository() IIndexRepository {
return &indexRepository{}
}
func (r *indexRepository) WipeIndex(ctx context.Context, dbContext persistence.IDbContext) error {
cmd := `DELETE FROM note;
DELETE FROM tag;
DELETE FROM note_tag;
DELETE FROM link;`
_, err := dbContext.ExecContext(ctx, cmd)
return err
}

20
persistence/repositories/link_repository.go
View file

@ -10,6 +10,7 @@ import (
type ILinkRepository interface {
Insert(ctx context.Context, dbContext persistence.IDbContext, links []*models.Link) error
InsertRange(ctx context.Context, dbContext persistence.IDbContext, links []*models.Link) error
GetByNoteID(ctx context.Context, dbContext persistence.IDbContext, fromNoteID string) ([]*models.Link, error)
GetBySlug(ctx context.Context, dbContext persistence.IDbContext, targetSlug string) ([]*models.Link, error)
Search(ctx context.Context, query string) ([]*models.Link, error)
@ -43,6 +44,25 @@ func (r *linkRepository) Insert(ctx context.Context, dbContext persistence.IDbCo
return err
}
func (r *linkRepository) InsertRange(ctx context.Context, dbContext persistence.IDbContext, links []*models.Link) error {
if len(links) == 0 {
return nil
}
linkStatement, err := dbContext.Prepare(`INSERT OR IGNORE INTO link (id, from_note_id, target_slug, raw, display, line, col) VALUES (?, ?, ?, ?, ?, ?, ?)`)
if err != nil {
return err
}
for _, link := range links {
if _, err := linkStatement.ExecContext(ctx, link.ID(), link.FromNoteID(), link.TargetSlug(), link.Raw(), link.Display(), link.Line(), link.Col()); err != nil {
return err
}
}
return nil
}
func (r *linkRepository) GetByNoteID(ctx context.Context, dbContext persistence.IDbContext, fromNoteID string) ([]*models.Link, error) {
rows, err := dbContext.QueryContext(ctx, "SELECT id, from_note_id, target_slug, raw, display, line, col FROM link WHERE from_note_id = ?", fromNoteID)
if err != nil {

21
persistence/repositories/note_repository.go
View file

@ -11,11 +11,12 @@ import (
type INoteRepository interface {
Insert(ctx context.Context, dbContext persistence.IDbContext, note *models.Note) error
InsertRange(ctx context.Context, dbContext persistence.IDbContext, note []*models.Note) error
Update(ctx context.Context, dbContext persistence.IDbContext, noteID, path, title, slug string) error
GetBySlug(ctx context.Context, slug string) (*models.Note, error)
}
type noteRepository struct{
type noteRepository struct {
readDBContext persistence.ReadContext
}
@ -35,6 +36,24 @@ func (r *noteRepository) Insert(ctx context.Context, dbContext persistence.IDbCo
return err
}
func (r *noteRepository) InsertRange(ctx context.Context, dbContext persistence.IDbContext, notes []*models.Note) error {
if len(notes) == 0 {
return nil
}
noteStatement, err := dbContext.Prepare(`
INSERT INTO note (id, title, path, slug, created_at, updated_at)
VALUES (?, ?, ?, ?, datetime('now'), datetime('now'))`)
if err != nil {
return err
}
for _, note := range notes {
if _, err := noteStatement.ExecContext(ctx, note.ID(), note.Title(), note.Path(), note.Slug()); err != nil {
return err
}
}
return nil
}
func (r *noteRepository) Update(ctx context.Context, dbContext persistence.IDbContext, noteID, path, title, slug string) error {
query := `
UPDATE note

38
persistence/repositories/tag_repository.go
View file

@ -10,7 +10,8 @@ import (
type ITagRepository interface {
Insert(ctx context.Context, dbCtx persistence.IDbContext, tags []*models.Tag) error
InsertNoteTags(ctx context.Context, dbCtx persistence.IDbContext, noteID string, tagIDs []string) error
InsertRange(ctx context.Context, dbCtx persistence.IDbContext, tags []*models.Tag) error
InsertNoteTags(ctx context.Context, dbCtx persistence.IDbContext, noteTags []*models.NoteTag) error
GetByNames(ctx context.Context, dbCtx persistence.IDbContext, names []string) ([]*models.Tag, error)
DeleteNoteTags(ctx context.Context, dbCtx persistence.IDbContext, noteID string) error
}
@ -40,26 +41,37 @@ func (r *tagRepository) Insert(ctx context.Context, dbCtx persistence.IDbContext
return err
}
func (r *tagRepository) InsertNoteTags(ctx context.Context, dbCtx persistence.IDbContext, noteID string, tags []string) error {
func (r *tagRepository) InsertRange(ctx context.Context, dbCtx persistence.IDbContext, tags []*models.Tag) error {
if len(tags) == 0 {
return nil
}
placeholders := make([]string, 0, len(tags))
args := make([]any, 0, len(tags))
for _, tagID := range tags {
placeholders = append(placeholders, "(?, ?)")
args = append(args, noteID, tagID)
}
query := "INSERT OR IGNORE INTO note_tag(note_id, tag_id) VALUES " + strings.Join(placeholders, ",")
_, err := dbCtx.ExecContext(ctx, query, args...)
tagStatement, err := dbCtx.Prepare(`INSERT OR IGNORE INTO tag (name) VALUES (?)`)
if err != nil {
return err
}
for _, tag := range tags {
if _, err = tagStatement.ExecContext(ctx, tag.Name()); err != nil {
return err
}
}
return nil
}
func (r *tagRepository) InsertNoteTags(ctx context.Context, dbCtx persistence.IDbContext, noteTags []*models.NoteTag) error {
noteTagStmt, err := dbCtx.Prepare(`INSERT INTO note_tag (note_id, tag_id) VALUES (?, ?)`)
if err != nil {
return err
}
for _, noteTag := range noteTags {
if _, err = noteTagStmt.ExecContext(ctx, noteTag.NoteID(), noteTag.TagID()); err != nil {
return err
}
}
return nil
}