feat: implement core backend architecture and project management services for the History API

pkg/ai/rag.go

@@ -19,6 +19,8 @@ type RagUtils struct {
 	embedder *embeddings.EmbedderImpl
 }
 
+var htmlTagRegex = regexp.MustCompile(`<[^>]*>`)
+
 func NewRagUtils() (*RagUtils, error) {
 	openRouterAPIKey, err := config.GetConfig("OPEN_ROUTER_API")
 	if err != nil {
@@ -57,8 +59,7 @@ func NewRagUtils() (*RagUtils, error) {
 }
 
 func (u *RagUtils) StripHTML(text string) string {
-	re := regexp.MustCompile(`<[^>]*>`)
-	text = re.ReplaceAllString(text, " ")
+	text = htmlTagRegex.ReplaceAllString(text, " ")
 	return html.UnescapeString(text)
 }
 

pkg/cache/keys.go

@@ -0,0 +1,22 @@
+package cache
+
+import (
+	"strconv"
+
+	"github.com/cespare/xxhash/v2"
+
+	"history-api/pkg/jsonx"
+)
+
+func Key(prefix, id string) string {
+	return prefix + ":" + id
+}
+
+func Key2(prefix, first, second string) string {
+	return prefix + ":" + first + ":" + second
+}
+
+func QueryKey(prefix string, params any) string {
+	data, _ := jsonx.Marshal(params)
+	return prefix + ":" + strconv.FormatUint(xxhash.Sum64(data), 16)
+}

pkg/cache/redis.go

@@ -2,10 +2,11 @@ package cache
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"history-api/pkg/config"
 	"history-api/pkg/constants"
+	json "history-api/pkg/jsonx"
+	"runtime"
 	"time"
 
 	"github.com/redis/go-redis/v9"
@@ -27,16 +28,39 @@ type RedisClient struct {
 	client *redis.Client
 }
 
+func defaultRedisPoolSize() int {
+	poolSize := runtime.NumCPU() * 32
+	if poolSize < 64 {
+		return 64
+	}
+	if poolSize > 256 {
+		return 256
+	}
+	return poolSize
+}
+
 func NewRedisClient() (Cache, error) {
 	uri, err := config.GetConfig("REDIS_CONNECTION_URI")
 	if err != nil {
 		return nil, err
 	}
 
+	poolSize := config.GetIntConfigWithDefault("REDIS_POOL_SIZE", defaultRedisPoolSize())
+	if poolSize < 1 {
+		poolSize = defaultRedisPoolSize()
+	}
+	minIdleConns := config.GetIntConfigWithDefault("REDIS_MIN_IDLE_CONNS", poolSize/8)
+	if minIdleConns < 0 {
+		minIdleConns = 0
+	}
+	if minIdleConns > poolSize {
+		minIdleConns = poolSize
+	}
+
 	rdb := redis.NewClient(&redis.Options{
 		Addr:         uri,
-		PoolSize:     500,
-		MinIdleConns: 50,
+		PoolSize:     poolSize,
+		MinIdleConns: minIdleConns,
 		DialTimeout:  5 * time.Second,
 		ReadTimeout:  3 * time.Second,
 		WriteTimeout: 3 * time.Second,
@@ -83,8 +107,8 @@ func (r *RedisClient) DelByPattern(ctx context.Context, pattern string) error {
 
 		if len(keys) > 0 {
 			if err := r.client.Unlink(ctx, keys...).Err(); err != nil {
-                return fmt.Errorf("error unlinking keys during scan: %v", err)
-            }
+				return fmt.Errorf("error unlinking keys during scan: %v", err)
+			}
 		}
 
 		cursor = nextCursor
@@ -155,7 +179,7 @@ func (r *RedisClient) PublishTask(ctx context.Context, streamName string, taskTy
 
 func GetMultiple[T any](ctx context.Context, c Cache, keys []string) ([]T, error) {
 	raws := c.MGet(ctx, keys...)
-	final := make([]T, 0)
+	final := make([]T, 0, len(raws))
 	for _, b := range raws {
 		if b == nil {
 			continue

pkg/config/config.go

@@ -1,10 +1,10 @@
 package config
 
 import (
-	"errors"
 	"fmt"
 	"history-api/assets"
 	"os"
+	"strconv"
 	"strings"
 
 	"github.com/joho/godotenv"
@@ -13,15 +13,17 @@ import (
 func LoadEnv() error {
 	envData, err := assets.GetFileContent("resources/.env")
 	if err != nil {
-		return errors.New("error read .env file")
+		return nil
 	}
 	envMap, err := godotenv.Parse(strings.NewReader(envData))
 	if err != nil {
-		return errors.New("error parsing .env content")
+		return fmt.Errorf("error parsing .env content: %w", err)
 	}
 
 	for key, value := range envMap {
-		os.Setenv(key, value)
+		if os.Getenv(key) == "" {
+			os.Setenv(key, value)
+		}
 	}
 	return nil
 }
@@ -41,4 +43,30 @@ func GetConfigWithDefault(config, defaultValue string) string {
 		return defaultValue
 	}
 	return data
-}
+}
+
+func GetIntConfigWithDefault(config string, defaultValue int) int {
+	data := strings.TrimSpace(os.Getenv(config))
+	if data == "" {
+		return defaultValue
+	}
+
+	value, err := strconv.Atoi(data)
+	if err != nil {
+		return defaultValue
+	}
+	return value
+}
+
+func GetBoolConfigWithDefault(config string, defaultValue bool) bool {
+	data := strings.TrimSpace(os.Getenv(config))
+	if data == "" {
+		return defaultValue
+	}
+
+	value, err := strconv.ParseBool(data)
+	if err != nil {
+		return defaultValue
+	}
+	return value
+}

pkg/convert/convert.go

@@ -1,18 +1,14 @@
 package convert
 
 import (
-	"crypto/md5"
-	"encoding/json"
-	"fmt"
+	"history-api/pkg/cache"
 	"time"
 
 	"github.com/jackc/pgx/v5/pgtype"
 )
 
 func GenerateQueryKey(prefix string, params any) string {
-	b, _ := json.Marshal(params)
-	hash := fmt.Sprintf("%x", md5.Sum(b))
-	return fmt.Sprintf("%s:query:%s", prefix, hash)
+	return cache.QueryKey(prefix, params)
 }
 
 func UUIDToString(v pgtype.UUID) string {

pkg/database/db.go

@@ -3,11 +3,23 @@ package database
 import (
 	"context"
 	"history-api/pkg/config"
+	"runtime"
 	"time"
 
 	"github.com/jackc/pgx/v5/pgxpool"
 )
 
+func defaultMaxConns() int {
+	conns := runtime.NumCPU() * 8
+	if conns < 16 {
+		return 16
+	}
+	if conns > 64 {
+		return 64
+	}
+	return conns
+}
+
 func NewPostgresqlDB() (*pgxpool.Pool, error) {
 	ctx := context.Background()
 
@@ -20,8 +32,23 @@ func NewPostgresqlDB() (*pgxpool.Pool, error) {
 	if err != nil {
 		return nil, err
 	}
-	poolConfig.MaxConns = 100
-	poolConfig.MinConns = 10
+	maxConns := config.GetIntConfigWithDefault("PGX_MAX_CONNS", defaultMaxConns())
+	if maxConns < 1 {
+		maxConns = defaultMaxConns()
+	}
+
+	minConns := config.GetIntConfigWithDefault("PGX_MIN_CONNS", maxConns/4)
+	if minConns < 0 {
+		minConns = 0
+	}
+	if minConns > maxConns {
+		minConns = maxConns
+	}
+
+	poolConfig.MaxConns = int32(maxConns)
+	poolConfig.MinConns = int32(minConns)
+	poolConfig.MaxConnIdleTime = time.Duration(config.GetIntConfigWithDefault("PGX_MAX_CONN_IDLE_SECONDS", 300)) * time.Second
+	poolConfig.HealthCheckPeriod = time.Duration(config.GetIntConfigWithDefault("PGX_HEALTH_CHECK_SECONDS", 60)) * time.Second
 
 	var pool *pgxpool.Pool
 

pkg/jsonx/json.go

@@ -0,0 +1,11 @@
+package jsonx
+
+import "github.com/bytedance/sonic"
+
+func Marshal(v any) ([]byte, error) {
+	return sonic.Marshal(v)
+}
+
+func Unmarshal(data []byte, v any) error {
+	return sonic.Unmarshal(data, v)
+}

pkg/mbtiles/db.go

@@ -1,26 +1,40 @@
 package mbtiles
 
 import (
-    "database/sql"
-    "fmt"
+	"database/sql"
+	"fmt"
+	"history-api/pkg/config"
+	"runtime"
 
-    _ "github.com/glebarez/go-sqlite"
+	_ "github.com/glebarez/go-sqlite"
 )
 
 func NewMBTilesDB(path string) (*sql.DB, error) {
-    dsn := fmt.Sprintf("file:%s?mode=ro&_journal_mode=off&_synchronous=off", path)
-    db, err := sql.Open("sqlite", dsn) 
-    if err != nil {
-        return nil, err
-    }
+	dsn := fmt.Sprintf("file:%s?mode=ro&_journal_mode=off&_synchronous=off", path)
+	db, err := sql.Open("sqlite", dsn)
+	if err != nil {
+		return nil, err
+	}
 
-    err = db.Ping()
-    if err != nil {
-        return nil, err
-    }
+	err = db.Ping()
+	if err != nil {
+		return nil, err
+	}
 
-    db.SetMaxOpenConns(10)
-    db.SetMaxIdleConns(5)
+	maxOpenConns := config.GetIntConfigWithDefault("MBTILES_MAX_OPEN_CONNS", runtime.NumCPU()*4)
+	if maxOpenConns < 1 {
+		maxOpenConns = 1
+	}
+	maxIdleConns := config.GetIntConfigWithDefault("MBTILES_MAX_IDLE_CONNS", maxOpenConns/2)
+	if maxIdleConns < 1 {
+		maxIdleConns = 1
+	}
+	if maxIdleConns > maxOpenConns {
+		maxIdleConns = maxOpenConns
+	}
 
-    return db, nil
-}
+	db.SetMaxOpenConns(maxOpenConns)
+	db.SetMaxIdleConns(maxIdleConns)
+
+	return db, nil
+}

pkg/storage/s3.go

@@ -212,7 +212,7 @@ func (s *s3Storage) BulkDelete(ctx context.Context, keys []string) error {
 		}
 
 		batch := keys[i:end]
-		var objects []types.ObjectIdentifier
+		objects := make([]types.ObjectIdentifier, 0, len(batch))
 		for _, k := range batch {
 			objects = append(objects, types.ObjectIdentifier{Key: aws.String(k)})
 		}

pkg/validator/validator.go

@@ -107,9 +107,10 @@ type ErrorResponse struct {
 
 func formatValidationError(err error) []*ErrorResponse {
 	var validationErrors validator.ValidationErrors
-	var errorsList []*ErrorResponse
+	errorsList := make([]*ErrorResponse, 0, 1)
 
 	if errors.As(err, &validationErrors) {
+		errorsList = make([]*ErrorResponse, 0, len(validationErrors))
 		for _, fieldError := range validationErrors {
 			message := ""
 			switch fieldError.Tag() {