init

hdiff-any-game/Makefile

@@ -0,0 +1,12 @@
+APP_NAME = hdiff-any-game.exe
+
+all: build
+
+build:
+	go build -o $(APP_NAME) ./
+
+run: build
+	./$(APP_NAME)
+
+clean:
+	rm -f $(APP_NAME) diff.txt

hdiff-any-game/checker.go

@@ -0,0 +1,138 @@
+package main
+
+import (
+	"crypto/md5"
+	"encoding/binary"
+	"encoding/hex"
+	"io"
+	"os"
+	"runtime"
+	"sync"
+
+	"github.com/schollz/progressbar/v3"
+)
+
+type DiffResult struct {
+	OnlyInOld []string
+	OnlyInNew []string
+	Changed   []string
+}
+
+func safePartialMD5(path string) (string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	fi, err := f.Stat()
+	if err != nil {
+		return "", err
+	}
+	size := fi.Size()
+	modTime := fi.ModTime().UnixNano()
+
+	h := md5.New()
+
+	binary.Write(h, binary.LittleEndian, size)
+	binary.Write(h, binary.LittleEndian, modTime)
+	binary.Write(h, binary.LittleEndian, fi.Mode())
+
+	buf := make([]byte, 4096)
+
+	if size <= 16*1024 {
+		f.Seek(0, io.SeekStart)
+		io.Copy(h, f)
+		return hex.EncodeToString(h.Sum(nil)), nil
+	}
+
+	n, _ := f.Read(buf)
+	h.Write(buf[:n])
+
+	if size > 8192 {
+		mid := size / 2
+		f.Seek(mid-2048, io.SeekStart)
+		n, _ = f.Read(buf)
+		h.Write(buf[:n])
+	}
+
+	if size > 4096 {
+		f.Seek(-4096, io.SeekEnd)
+		n, _ = f.Read(buf)
+		h.Write(buf[:n])
+	}
+
+	return hex.EncodeToString(h.Sum(nil)), nil
+}
+
+func DiffFolders(oldPath, newPath string) (*DiffResult, error) {
+	oldFiles, err := collectFiles(oldPath)
+	if err != nil {
+		return nil, err
+	}
+	newFiles, err := collectFiles(newPath)
+	if err != nil {
+		return nil, err
+	}
+
+	result := &DiffResult{}
+	var mu sync.Mutex
+	var wg sync.WaitGroup
+
+	jobs := make(chan [3]string, 100)
+
+	total := 0
+	for rel := range oldFiles {
+		if _, ok := newFiles[rel]; ok {
+			total++
+		}
+	}
+
+	bar := progressbar.NewOptions(total,
+		progressbar.OptionSetDescription("🔍 Comparing files"),
+		progressbar.OptionSetWidth(30),
+		progressbar.OptionShowCount(),
+		progressbar.OptionSetPredictTime(true),
+		progressbar.OptionShowElapsedTimeOnFinish(),
+	)
+
+	workers := runtime.NumCPU() * 2
+
+	for i := 0; i < workers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for job := range jobs {
+				rel, oldFile, newFile := job[0], job[1], job[2]
+				oldHash, _ := safePartialMD5(oldFile)
+				newHash, _ := safePartialMD5(newFile)
+				if oldHash != newHash {
+					mu.Lock()
+					result.Changed = append(result.Changed, rel)
+					mu.Unlock()
+				}
+				bar.Add(1)
+			}
+		}()
+	}
+
+	for rel, oldFile := range oldFiles {
+		if newFile, ok := newFiles[rel]; ok {
+			jobs <- [3]string{rel, oldFile, newFile}
+		} else {
+			result.OnlyInOld = append(result.OnlyInOld, rel)
+		}
+	}
+
+	for rel := range newFiles {
+		if _, ok := oldFiles[rel]; !ok {
+			result.OnlyInNew = append(result.OnlyInNew, rel)
+		}
+	}
+
+	close(jobs)
+	wg.Wait()
+	bar.Finish()
+
+	return result, nil
+}

hdiff-any-game/copyNewFIle.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/schollz/progressbar/v3"
+)
+
+func CopyNewFiles(newPath string, result *DiffResult) error {
+	delFile, err := os.Create("hdiff/deletefiles.txt")
+	if err != nil {
+		return err
+	}
+	defer delFile.Close()
+	for _, f := range result.OnlyInOld {
+		fmt.Fprintln(delFile, f)
+	}
+
+	bar := progressbar.NewOptions(len(result.OnlyInNew),
+		progressbar.OptionSetDescription("📂 Copying new files"),
+		progressbar.OptionSetWidth(30),
+		progressbar.OptionShowCount(),
+		progressbar.OptionSetPredictTime(true),
+	)
+
+	for _, rel := range result.OnlyInNew {
+		src := filepath.Join(newPath, rel)
+		dst := filepath.Join("hdiff", rel)
+		os.MkdirAll(filepath.Dir(dst), 0755)
+
+		if err := copyFile(src, dst); err != nil {
+			fmt.Println("copy error:", err)
+		}
+		bar.Add(1)
+	}
+	bar.Finish()
+
+	return nil
+}

hdiff-any-game/go.mod

@@ -0,0 +1,11 @@
+module hdiff-any-game
+
+go 1.25.0
+
+require (
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/schollz/progressbar/v3 v3.18.0 // indirect
+	golang.org/x/sys v0.35.0 // indirect
+	golang.org/x/term v0.34.0 // indirect
+)

hdiff-any-game/go.sum

@@ -0,0 +1,10 @@
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA=
+github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
+golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
+golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4=
+golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw=

hdiff-any-game/hdiffz.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"os/exec"
+	"path/filepath"
+)
+
+func runHdiffz(oldPath, newPath, outDiff string) error {
+	args := []string{"-s-64", "-SD", "-c-zstd-21-24", "-d", oldPath, newPath, outDiff}
+	hdiffzPath, err := filepath.Abs(filepath.Join("bin", "hdiffz.exe"))
+	if err != nil {
+		return err
+	}
+	cmd := exec.Command(hdiffzPath, args...)
+	cmd.Stdout = nil
+	cmd.Stderr = nil
+
+	return cmd.Run()
+}

hdiff-any-game/main.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"bufio"
+	"embed"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+//go:embed bin/hdiffz.exe
+//go:embed bin/7za.exe
+var embeddedFiles embed.FS
+
+func ensureBinaries() (map[string]string, error) {
+	binDir := "bin"
+	if _, err := os.Stat(binDir); os.IsNotExist(err) {
+		if err := os.MkdirAll(binDir, 0755); err != nil {
+			return nil, err
+		}
+	}
+
+	files := []string{"hdiffz.exe", "7za.exe"}
+	paths := make(map[string]string)
+
+	for _, f := range files {
+		destPath := filepath.Join(binDir, f)
+		paths[f] = destPath
+
+		if _, err := os.Stat(destPath); os.IsNotExist(err) {
+			data, err := embeddedFiles.ReadFile("bin/" + f)
+			if err != nil {
+				return nil, err
+			}
+			if err := os.WriteFile(destPath, data, 0755); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return paths, nil
+}
+
+func main() {
+	paths, err := ensureBinaries()
+	if err != nil {
+		fmt.Println("Error:", err)
+		return
+	}
+
+	for _, path := range paths {
+		fmt.Println("Binary ready at:", path)
+	}
+	reader := bufio.NewReader(os.Stdin)
+
+	fmt.Print("Enter OLD game path: ")
+	oldPath, _ := reader.ReadString('\n')
+	oldPath = strings.TrimSpace(oldPath)
+	if oldPath == "" {
+		fmt.Fprintln(os.Stderr, "no old path provided")
+		os.Exit(1)
+	}
+
+	fmt.Print("Enter NEW game path: ")
+	newPath, _ := reader.ReadString('\n')
+	newPath = strings.TrimSpace(newPath)
+	if newPath == "" {
+		fmt.Fprintln(os.Stderr, "no new path provided")
+		os.Exit(1)
+	}
+
+	fmt.Print("Enter zip hdiff output name: ")
+	hdiffName, _ := reader.ReadString('\n')
+	hdiffName = strings.TrimSpace(hdiffName)
+	if hdiffName == "" {
+		fmt.Fprintln(os.Stderr, "no hdiff output provided")
+		os.Exit(1)
+	}
+
+	if !strings.HasSuffix(strings.ToLower(hdiffName), ".zip") {
+		hdiffName += ".zip"
+	}
+
+	result, err := DiffFolders(oldPath, newPath)
+	if err != nil {
+		fmt.Println("Error:", err)
+		return
+	}
+	hdiffFolderPath := filepath.Join(".", "hdiff")
+	os.MkdirAll(hdiffFolderPath, 0755)
+	if err := CopyNewFiles(newPath, result); err != nil {
+		fmt.Println("Error writing diff:", err)
+		return
+	}
+	if err := MakeHdiffFile(oldPath, newPath, result.Changed); err != nil {
+		fmt.Println("Error writing diff:", err)
+		return
+	}
+
+	if err := ZipWith7za(hdiffFolderPath, hdiffName); err != nil {
+		fmt.Println("Error writing diff:", err)
+		return
+	}
+
+	if err := RemoveFolderWithProgress(hdiffFolderPath); err != nil {
+		fmt.Fprintln(os.Stderr, "error removing temp dir:", err)
+		os.Exit(1)
+	}
+
+	fmt.Println("Done")
+}

hdiff-any-game/makeHdiffFile.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sync"
+
+	"github.com/schollz/progressbar/v3"
+)
+
+type HdiffFile struct {
+	RemoteName string `json:"remoteName"`
+}
+
+func MakeHdiffFile(oldPath string, newPath string, changedFiles []string) error {
+	delFile, err := os.Create("hdiff/hdifffiles.txt")
+	if err != nil {
+		return err
+	}
+	defer delFile.Close()
+
+	for _, f := range changedFiles {
+		data, err := json.Marshal(HdiffFile{RemoteName: f})
+		if err != nil {
+			return err
+		}
+		fmt.Fprintln(delFile, string(data))
+	}
+
+	bar := progressbar.NewOptions(len(changedFiles),
+		progressbar.OptionSetDescription("Creating HDIFF files"),
+		progressbar.OptionShowCount(),
+		progressbar.OptionSetWidth(30),
+		progressbar.OptionSetPredictTime(true),
+	)
+
+	workers := runtime.NumCPU() / 2
+	if workers < 2 {
+		workers = 2
+	}
+	jobs := make(chan string, len(changedFiles))
+	var wg sync.WaitGroup
+
+	for i := int64(0); i < int64(workers); i++ {
+		wg.Go(func() {
+			for f := range jobs {
+				oldFile := filepath.Join(oldPath, f)
+				newFile := filepath.Join(newPath, f)
+				hdiffPath := filepath.Join("hdiff", f+".hdiff")
+				if err := os.MkdirAll(filepath.Dir(hdiffPath), 0755); err != nil {
+					fmt.Fprintf(os.Stderr, "failed to create dir: %v\n", err)
+					continue
+				}
+				runHdiffz(oldFile, newFile, hdiffPath)
+				bar.Add(1)
+			}
+		})
+	}
+
+	for _, f := range changedFiles {
+		jobs <- f
+	}
+	close(jobs)
+
+	wg.Wait()
+	bar.Finish()
+	return nil
+}

hdiff-any-game/utils.go

@@ -0,0 +1,161 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"sync"
+
+	"github.com/schollz/progressbar/v3"
+)
+
+func collectFiles(root string) (map[string]string, error) {
+	files := sync.Map{}
+	var wg sync.WaitGroup
+	dirs := make(chan string, 100)
+
+	workers := runtime.NumCPU() * 2
+	if workers < 1 {
+		workers = 1
+	}
+
+	for i := 0; i < workers; i++ {
+		go func() {
+			for dir := range dirs {
+				entries, err := os.ReadDir(dir)
+				if err != nil {
+					wg.Done()
+					continue
+				}
+				for _, e := range entries {
+					path := filepath.Join(dir, e.Name())
+					if e.IsDir() {
+						wg.Add(1)
+						dirs <- path
+					} else {
+						rel, _ := filepath.Rel(root, path)
+						files.Store(filepath.ToSlash(rel), path)
+					}
+				}
+				wg.Done()
+			}
+		}()
+	}
+	wg.Add(1)
+	dirs <- root
+
+	go func() {
+		wg.Wait()
+		close(dirs)
+	}()
+
+	wg.Wait()
+	out := make(map[string]string)
+	files.Range(func(k, v any) bool {
+		out[k.(string)] = v.(string)
+		return true
+	})
+
+	return out, nil
+}
+
+func copyFile(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, in)
+	return err
+}
+
+func ZipWith7za(src, dest string) error {
+	if _, err := os.Stat(src); os.IsNotExist(err) {
+		return fmt.Errorf("source folder does not exist: %s", src)
+	}
+
+	if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
+		return err
+	}
+
+	files, err := os.ReadDir(src)
+	if err != nil {
+		return err
+	}
+
+	if len(files) == 0 {
+		return fmt.Errorf("source folder is empty: %s", src)
+	}
+
+	sevenZipPath, err := filepath.Abs(filepath.Join("bin", "7za.exe"))
+	if err != nil {
+		return err
+	}
+
+	destAbs, err := filepath.Abs(filepath.Join(".", dest))
+	if err != nil {
+		return err
+	}
+
+	args := []string{"a", "-tzip", "-mx=1", "-mmt=on", destAbs}
+	for _, f := range files {
+		args = append(args, f.Name())
+	}
+
+	cmd := exec.Command(sevenZipPath, args...)
+	cmd.Dir = src
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	return cmd.Run()
+}
+
+func RemoveFolderWithProgress(folder string) error {
+	var total int
+	filepath.Walk(folder, func(path string, info os.FileInfo, err error) error {
+		if err == nil {
+			total++
+		}
+		return nil
+	})
+
+	bar := progressbar.NewOptions(total,
+		progressbar.OptionSetDescription("Removing temp files"),
+		progressbar.OptionShowCount(),
+		progressbar.OptionSetWidth(30),
+		progressbar.OptionSetPredictTime(true),
+	)
+
+	err := filepath.Walk(folder, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			if err := os.Remove(path); err != nil {
+				return err
+			}
+			bar.Add(1)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	if err := os.RemoveAll(folder); err != nil {
+		return err
+	}
+
+	bar.Finish()
+	fmt.Println("\nTemp folder removed")
+	return nil
+}