Update progress bar and more embed

ldiff-converter/go.mod

@@ -3,7 +3,16 @@ module ldiff-converter
 go 1.25.0
 
 require (
+	github.com/amenzhinsky/go-memexec v0.7.1
 	github.com/klauspost/compress v1.18.0
 	golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b
 	google.golang.org/protobuf v1.36.8
 )
+
+require (
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/schollz/progressbar/v3 v3.18.0 // indirect
+	golang.org/x/sys v0.29.0 // indirect
+	golang.org/x/term v0.28.0 // indirect
+)

ldiff-converter/go.sum

@@ -1,6 +1,20 @@
+github.com/amenzhinsky/go-memexec v0.7.1 h1:DVm4cXzklaNWZoTJgZUi/dlXtelhC7QBtX4luKjl1qk=
+github.com/amenzhinsky/go-memexec v0.7.1/go.mod h1:ApTO9/i2bcii7kvIXi74gum+/zYDzkiOXtuBZoYOKVE=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA=
+github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
 golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b h1:DXr+pvt3nC887026GRP39Ej11UATqWDmWuS99x26cD0=
 golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b/go.mod h1:4QTo5u+SEIbbKW1RacMZq1YEfOBqeXa19JeshGi+zc4=
+golang.org/x/sys v0.0.0-20210903071746-97244b99971b h1:3Dq0eVHn0uaQJmPO+/aYPI/fRMqdrVDbu7MQcku54gg=
+golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
+golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=
+golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
 google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
 google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=

ldiff-converter/ldiff.go

@@ -1,15 +1,15 @@
 package main
 
 import (
-	"ldiff-converter/pb"
 	"fmt"
+	"io"
+	"ldiff-converter/pb"
 	"os"
 	"path/filepath"
 
 	"golang.org/x/exp/mmap"
 )
 
-
 func LDiffFile(data *pb.AssetManifest, assetName string, assetSize int64, ldiffsDir, outputDir string) error {
 	path := filepath.Join(ldiffsDir, data.ChunkFileName)
 
@@ -17,15 +17,25 @@ func LDiffFile(data *pb.AssetManifest, assetName string, assetSize int64, ldiffs
 	if err != nil {
 		return fmt.Errorf("%s does not exist: %w", path, err)
 	}
-
 	fileSize := info.Size()
+
 	var buffer []byte
 
 	if fileSize > 10*1024*1024 && data.HdiffFileSize > 1*1024*1024 {
-		// mmap for large files using x/exp/mmap
+		// ưu tiên mmap cho file lớn
 		reader, err := mmap.Open(path)
-		if err != nil {
-			// fallback to buffered read
+		if err == nil {
+			defer reader.Close()
+			buffer = make([]byte, data.HdiffFileSize)
+			n, err := reader.ReadAt(buffer, data.HdiffFileInChunkOffset)
+			if err != nil && err != io.EOF {
+				return fmt.Errorf("error reading mmap data: %w", err)
+			}
+			if int64(n) < data.HdiffFileSize {
+				return fmt.Errorf("expected %d bytes, but read %d bytes", data.HdiffFileSize, n)
+			}
+		} else {
+			// fallback sang buffered read
 			file, err := os.Open(path)
 			if err != nil {
 				return fmt.Errorf("error opening file %s: %w", path, err)
@@ -35,22 +45,14 @@ func LDiffFile(data *pb.AssetManifest, assetName string, assetSize int64, ldiffs
 			if err != nil {
 				return err
 			}
-		} else {
-			defer reader.Close()
-			buffer = make([]byte, data.HdiffFileSize)
-			_, err := reader.ReadAt(buffer, data.HdiffFileInChunkOffset)
-			if err != nil {
-				return fmt.Errorf("error reading mmap data: %w", err)
-			}
 		}
 	} else {
-		// small files, buffered read
+		// file nhỏ, dùng buffered read
 		file, err := os.Open(path)
 		if err != nil {
 			return fmt.Errorf("error opening file %s: %w", path, err)
 		}
 		defer file.Close()
-
 		buffer, err = ReadBuffer(file, data.HdiffFileInChunkOffset, data.HdiffFileSize)
 		if err != nil {
 			return err
@@ -58,7 +60,7 @@ func LDiffFile(data *pb.AssetManifest, assetName string, assetSize int64, ldiffs
 	}
 
 	extension := ""
-	if data.OriginalFileSize != 0 || assetSize != data.HdiffFileSize {
+	if data.OriginalFileSize > 0 && assetSize != data.HdiffFileSize {
 		extension = ".hdiff"
 	}
 	assetPath := filepath.Join(outputDir, assetName+extension)
@@ -76,19 +78,16 @@ func LDiffFile(data *pb.AssetManifest, assetName string, assetSize int64, ldiffs
 	return nil
 }
 
-
-
 func ReadBuffer(file *os.File, offset int64, size int64) ([]byte, error) {
 	buffer := make([]byte, size)
 
 	n, err := file.ReadAt(buffer, offset)
-	if err != nil {
+	if err != nil && err != io.EOF {
 		return nil, fmt.Errorf("error reading data: %w", err)
 	}
-	if int64(n) != size {
+	if int64(n) < size {
 		return nil, fmt.Errorf("expected %d bytes, but read %d bytes", size, n)
 	}
 
 	return buffer, nil
 }
-

ldiff-converter/main.go

@@ -2,18 +2,23 @@ package main
 
 import (
 	"bufio"
+	_ "embed"
 	"encoding/json"
 	"fmt"
 	"ldiff-converter/pb"
 	"os"
 	"path/filepath"
 	"strings"
+
+	"github.com/schollz/progressbar/v3"
 )
 
-func main() {
+//go:embed bin/7za.exe
+var sevenZip []byte
 
+func main() {
 	reader := bufio.NewReader(os.Stdin)
-	fmt.Print("Enter zip ldiff path: ")
+	fmt.Print("Enter ldiff path: ")
 	ldiff, _ := reader.ReadString('\n')
 	ldiff = strings.TrimSpace(ldiff)
 	if ldiff == "" {
@@ -28,6 +33,9 @@ func main() {
 		fmt.Fprintln(os.Stderr, "no hdiff output provided")
 		os.Exit(1)
 	}
+	if !strings.HasSuffix(strings.ToLower(hdiff), ".zip") {
+		hdiff += ".zip"
+	}
 
 	tmpFolderPath := filepath.Join(".", "temp")
 	if err := os.MkdirAll(tmpFolderPath, 0755); err != nil {
@@ -35,20 +43,24 @@ func main() {
 		os.Exit(1)
 	}
 
-	if err := Unzip(ldiff, tmpFolderPath); err != nil {
+	fmt.Println("Unzipping ldiff...")
+	if err := UnzipWith7za(ldiff, tmpFolderPath); err != nil {
 		fmt.Fprintln(os.Stderr, "error:", err)
 		os.Exit(1)
 	}
+	fmt.Println("Unzipping ldiff done.")
 
 	ldiffPath := filepath.Join(tmpFolderPath, "ldiff")
 	manifestPath := filepath.Join(tmpFolderPath, "manifest")
 	hdiffFolderPath := filepath.Join(".", "hdiff")
 
+	fmt.Println("Loading manifest proto...")
 	manifestProto, err := LoadManifestProto(manifestPath)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "error loading manifest proto:", err)
 		os.Exit(1)
 	}
+	fmt.Println("Loading manifest proto done.")
 
 	ldiffEntries, err := os.ReadDir(ldiffPath)
 	if err != nil {
@@ -56,6 +68,13 @@ func main() {
 		os.Exit(1)
 	}
 
+	bar := progressbar.NewOptions(len(ldiffEntries),
+		progressbar.OptionSetDescription("📦 Converting ldiff files"),
+		progressbar.OptionShowCount(),
+		progressbar.OptionSetWidth(30),
+		progressbar.OptionSetPredictTime(true),
+	)
+	fmt.Println("Converting ldiff files...")
 	for _, ldiffEntry := range ldiffEntries {
 		assetName := ldiffEntry.Name()
 		var matchingAssets []struct {
@@ -77,7 +96,7 @@ func main() {
 				}
 			}
 		}
-
+		bar.Add(1)
 		for _, ma := range matchingAssets {
 			err := LDiffFile(ma.Asset, ma.AssetName, ma.AssetSize, ldiffPath, hdiffFolderPath)
 			if err != nil {
@@ -85,12 +104,15 @@ func main() {
 			}
 		}
 	}
-
+	bar.Finish()
+	fmt.Println()
+	fmt.Println("Converting ldiff files done.")
 	diffMapNames := make([]string, len(ldiffEntries))
 	for i, e := range ldiffEntries {
 		diffMapNames[i] = e.Name()
 	}
 
+	fmt.Println("Making diff map...")
 	diffMapList, err := MakeDiffMap(manifestProto, diffMapNames)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "error making diff map:", err)
@@ -107,22 +129,31 @@ func main() {
 		fmt.Fprintln(os.Stderr, "error writing diff map:", err)
 		os.Exit(1)
 	}
+	fmt.Println("Making diff map done.")
 
-	if err := os.RemoveAll(tmpFolderPath); err != nil {
+	fmt.Println("Removing temp ldiff files...")
+	if err := RemoveFolderWithProgress(tmpFolderPath, "🗑️ Removing temp ldiff files"); err != nil {
 		fmt.Fprintln(os.Stderr, "error removing temp dir:", err)
 		os.Exit(1)
 	}
+	fmt.Println()
+	fmt.Println("Removing temp ldiff files done.")
 
-	if err := Zip(hdiffFolderPath, hdiff); err != nil {
+	fmt.Println("Zipping hdiff files...")
+	if err := ZipWith7za(hdiffFolderPath, hdiff); err != nil {
 		fmt.Fprintln(os.Stderr, "error zip hdiff:", err)
 		os.Exit(1)
 	}
+	fmt.Println("Zipping hdiff files done.")
 
-	if err := os.RemoveAll(hdiffFolderPath); err != nil {
+	fmt.Println("Removing hdiff temp files...")
+	if err := RemoveFolderWithProgress(hdiffFolderPath, "🗑️ Removing hdiff temp files"); err != nil {
 		fmt.Fprintln(os.Stderr, "error removing temp dir:", err)
 		os.Exit(1)
 	}
+	fmt.Println()
+	fmt.Println("Removing hdiff temp files done.")
 
-	fmt.Println("done!")
+	fmt.Println("Done!")
 
 }

ldiff-converter/utils.go

@@ -1,112 +1,90 @@
 package main
 
 import (
-	"archive/zip"
 	"fmt"
-	"io"
 	"ldiff-converter/pb"
 	"os"
 	"path/filepath"
-	"strings"
+	"time"
+
+	"github.com/amenzhinsky/go-memexec"
+	"github.com/schollz/progressbar/v3"
 )
 
-func Unzip(src, dest string) error {
-	r, err := zip.OpenReader(src)
+func UnzipWith7za(src, dest string) error {
+	if _, err := os.Stat(src); os.IsNotExist(err) {
+		return fmt.Errorf("source file does not exist: %s", src)
+	}
+
+	if err := os.MkdirAll(dest, 0755); err != nil {
+		return err
+	}
+
+	destAbs, err := filepath.Abs(dest)
 	if err != nil {
 		return err
 	}
-	defer r.Close()
 
-	for _, f := range r.File {
-		fpath := filepath.Join(dest, f.Name)
-		if !strings.HasPrefix(fpath, filepath.Clean(dest)+string(os.PathSeparator)) {
-			return fmt.Errorf("illegal file path: %s", fpath)
-		}
+	exe, err := memexec.New(sevenZip)
+	if err != nil {
+		return err
+	}
+	defer exe.Close()
 
-		if f.FileInfo().IsDir() {
-			if err := os.MkdirAll(fpath, f.Mode()); err != nil {
-				return err
-			}
-			continue
-		}
+	args := []string{"x", "-y", "-o" + destAbs, src}
 
-		if err := os.MkdirAll(filepath.Dir(fpath), 0755); err != nil {
-			return err
-		}
-
-		rc, err := f.Open()
-		if err != nil {
-			return err
-		}
-
-		outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
-		if err != nil {
-			rc.Close()
-			return err
-		}
-
-		_, err = io.Copy(outFile, rc)
-		rc.Close()
-		outFile.Close()
-		if err != nil {
-			return err
-		}
+	cmd := exe.Command(args...)
+	cmd.Stdout = nil
+	cmd.Stderr = nil
+	if err := cmd.Run(); err != nil {
+		return err
 	}
 	return nil
 }
 
-func Zip(src, dest string) error {
-	zipFile, err := os.Create(dest)
+func ZipWith7za(src, dest string) error {
+	if _, err := os.Stat(src); os.IsNotExist(err) {
+		return fmt.Errorf("source folder does not exist: %s", src)
+	}
+
+	if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
+		return err
+	}
+
+	files, err := os.ReadDir(src)
 	if err != nil {
 		return err
 	}
-	defer zipFile.Close()
 
-	zw := zip.NewWriter(zipFile)
-	defer zw.Close()
+	if len(files) == 0 {
+		return fmt.Errorf("source folder is empty: %s", src)
+	}
 
-	err = filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
-		relPath, err := filepath.Rel(src, path)
-		if err != nil {
-			return err
-		}
-
-		if info.IsDir() {
-			if relPath == "." {
-				return nil
-			}
-			_, err := zw.Create(relPath + "/")
-			return err
-		}
-
-		fh, err := zip.FileInfoHeader(info)
-		if err != nil {
-			return err
-		}
-		fh.Name = relPath
-		fh.Method = zip.Deflate
-
-		writer, err := zw.CreateHeader(fh)
-		if err != nil {
-			return err
-		}
-
-		file, err := os.Open(path)
-		if err != nil {
-			return err
-		}
-		defer file.Close()
-
-		_, err = io.Copy(writer, file)
+	destAbs, err := filepath.Abs(filepath.Join(".", dest))
+	if err != nil {
 		return err
-	})
+	}
 
-	return err
+	args := []string{"a", "-tzip", "-mx=1", "-mmt=on", destAbs}
+	for _, f := range files {
+		args = append(args, f.Name())
+	}
+
+	exe, err := memexec.New(sevenZip)
+	if err != nil {
+		return err
+	}
+	defer exe.Close()
+	cmd := exe.Command(args...)
+	cmd.Dir = src
+	cmd.Stdout = nil
+	cmd.Stderr = nil
+	if err := cmd.Run(); err != nil {
+		return err
+	}
+	return nil
 }
+
 func MakeDiffMap(manifest *pb.ManifestProto, chunkNames []string) ([]*HDiffData, error) {
 	var hdiffFiles []*HDiffData
 
@@ -140,3 +118,51 @@ func MakeDiffMap(manifest *pb.ManifestProto, chunkNames []string) ([]*HDiffData,
 
 	return hdiffFiles, nil
 }
+
+func RemoveFolderWithProgress(folder string, title string) error {
+	var total int
+	filepath.Walk(folder, func(path string, info os.FileInfo, err error) error {
+		if err == nil && !info.IsDir() {
+			total++
+		}
+		return nil
+	})
+
+	bar := progressbar.NewOptions(total,
+		progressbar.OptionSetDescription(title),
+		progressbar.OptionShowCount(),
+		progressbar.OptionSetWidth(30),
+		progressbar.OptionSetPredictTime(true),
+	)
+
+	_ = filepath.Walk(folder, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return nil
+		}
+		if info.IsDir() {
+			return nil
+		}
+
+		// Retry tối đa 10 lần khi xóa
+		var rmErr error
+		for i := 0; i < 10; i++ {
+			rmErr = os.Remove(path)
+			if rmErr == nil {
+				break
+			}
+			time.Sleep(300 * time.Millisecond)
+		}
+		if rmErr != nil {
+			fmt.Printf("⚠️ Could not remove %s: %v\n", path, rmErr)
+		}
+
+		bar.Add(1)
+		return nil
+	})
+
+	if err := os.RemoveAll(folder); err != nil {
+		return fmt.Errorf("failed to remove folder %s: %w", folder, err)
+	}
+	bar.Finish()
+	return nil
+}