Update progress bar and more embed
This commit is contained in:
4
.gitignore
vendored
Normal file
4
.gitignore
vendored
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
*.zip
|
||||||
|
*.rar
|
||||||
|
hdiff-any-game/hdiff-any-game.exe
|
||||||
|
ldiff-converter/ldiff-converter.exe
|
||||||
@@ -133,6 +133,5 @@ func DiffFolders(oldPath, newPath string) (*DiffResult, error) {
|
|||||||
close(jobs)
|
close(jobs)
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
bar.Finish()
|
bar.Finish()
|
||||||
|
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -36,6 +36,5 @@ func CopyNewFiles(newPath string, result *DiffResult) error {
|
|||||||
bar.Add(1)
|
bar.Add(1)
|
||||||
}
|
}
|
||||||
bar.Finish()
|
bar.Finish()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ module hdiff-any-game
|
|||||||
go 1.25.0
|
go 1.25.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/amenzhinsky/go-memexec v0.7.1 // indirect
|
||||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||||
github.com/rivo/uniseg v0.4.7 // indirect
|
github.com/rivo/uniseg v0.4.7 // indirect
|
||||||
github.com/schollz/progressbar/v3 v3.18.0 // indirect
|
github.com/schollz/progressbar/v3 v3.18.0 // indirect
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
github.com/amenzhinsky/go-memexec v0.7.1 h1:DVm4cXzklaNWZoTJgZUi/dlXtelhC7QBtX4luKjl1qk=
|
||||||
|
github.com/amenzhinsky/go-memexec v0.7.1/go.mod h1:ApTO9/i2bcii7kvIXi74gum+/zYDzkiOXtuBZoYOKVE=
|
||||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
|
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
|
||||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
|
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
|
||||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||||
|
|||||||
Binary file not shown.
@@ -1,19 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
)
|
|
||||||
|
|
||||||
func runHdiffz(oldPath, newPath, outDiff string) error {
|
|
||||||
args := []string{"-s-64", "-SD", "-c-zstd-21-24", "-d", oldPath, newPath, outDiff}
|
|
||||||
hdiffzPath, err := filepath.Abs(filepath.Join("bin", "hdiffz.exe"))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
cmd := exec.Command(hdiffzPath, args...)
|
|
||||||
cmd.Stdout = nil
|
|
||||||
cmd.Stderr = nil
|
|
||||||
|
|
||||||
return cmd.Run()
|
|
||||||
}
|
|
||||||
@@ -2,7 +2,7 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
"embed"
|
_ "embed"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -10,48 +10,12 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
//go:embed bin/hdiffz.exe
|
//go:embed bin/hdiffz.exe
|
||||||
|
var hdiffz []byte
|
||||||
|
|
||||||
//go:embed bin/7za.exe
|
//go:embed bin/7za.exe
|
||||||
var embeddedFiles embed.FS
|
var sevenZip []byte
|
||||||
|
|
||||||
func ensureBinaries() (map[string]string, error) {
|
|
||||||
binDir := "bin"
|
|
||||||
if _, err := os.Stat(binDir); os.IsNotExist(err) {
|
|
||||||
if err := os.MkdirAll(binDir, 0755); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
files := []string{"hdiffz.exe", "7za.exe"}
|
|
||||||
paths := make(map[string]string)
|
|
||||||
|
|
||||||
for _, f := range files {
|
|
||||||
destPath := filepath.Join(binDir, f)
|
|
||||||
paths[f] = destPath
|
|
||||||
|
|
||||||
if _, err := os.Stat(destPath); os.IsNotExist(err) {
|
|
||||||
data, err := embeddedFiles.ReadFile("bin/" + f)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(destPath, data, 0755); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return paths, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
paths, err := ensureBinaries()
|
|
||||||
if err != nil {
|
|
||||||
fmt.Println("Error:", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, path := range paths {
|
|
||||||
fmt.Println("Binary ready at:", path)
|
|
||||||
}
|
|
||||||
reader := bufio.NewReader(os.Stdin)
|
reader := bufio.NewReader(os.Stdin)
|
||||||
|
|
||||||
fmt.Print("Enter OLD game path: ")
|
fmt.Print("Enter OLD game path: ")
|
||||||
@@ -82,31 +46,48 @@ func main() {
|
|||||||
hdiffName += ".zip"
|
hdiffName += ".zip"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fmt.Println("Diffing folders...")
|
||||||
result, err := DiffFolders(oldPath, newPath)
|
result, err := DiffFolders(oldPath, newPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Println("Error:", err)
|
fmt.Println("Error:", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println("Diffing folders done.")
|
||||||
|
|
||||||
hdiffFolderPath := filepath.Join(".", "hdiff")
|
hdiffFolderPath := filepath.Join(".", "hdiff")
|
||||||
os.MkdirAll(hdiffFolderPath, 0755)
|
os.MkdirAll(hdiffFolderPath, 0755)
|
||||||
|
|
||||||
|
fmt.Println("Copying new files...")
|
||||||
if err := CopyNewFiles(newPath, result); err != nil {
|
if err := CopyNewFiles(newPath, result); err != nil {
|
||||||
fmt.Println("Error writing diff:", err)
|
fmt.Println("Error writing diff:", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println("Copying new files done.")
|
||||||
|
|
||||||
|
fmt.Println("Making hdiff files...")
|
||||||
if err := MakeHdiffFile(oldPath, newPath, result.Changed); err != nil {
|
if err := MakeHdiffFile(oldPath, newPath, result.Changed); err != nil {
|
||||||
fmt.Println("Error writing diff:", err)
|
fmt.Println("Error writing diff:", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println("Making hdiff files done.")
|
||||||
|
|
||||||
|
fmt.Println("Zipping hdiff files...")
|
||||||
if err := ZipWith7za(hdiffFolderPath, hdiffName); err != nil {
|
if err := ZipWith7za(hdiffFolderPath, hdiffName); err != nil {
|
||||||
fmt.Println("Error writing diff:", err)
|
fmt.Println("Error writing diff:", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
fmt.Println("Zipping hdiff files done.")
|
||||||
|
|
||||||
if err := RemoveFolderWithProgress(hdiffFolderPath); err != nil {
|
fmt.Println("Removing hdiff temp files...")
|
||||||
|
if err := RemoveFolderWithProgress(hdiffFolderPath, "🗑️ Removing hdiff temp files"); err != nil {
|
||||||
fmt.Fprintln(os.Stderr, "error removing temp dir:", err)
|
fmt.Fprintln(os.Stderr, "error removing temp dir:", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println("Removing hdiff temp files done.")
|
||||||
|
|
||||||
fmt.Println("Done")
|
fmt.Println("Done")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ func MakeHdiffFile(oldPath string, newPath string, changedFiles []string) error
|
|||||||
}
|
}
|
||||||
|
|
||||||
bar := progressbar.NewOptions(len(changedFiles),
|
bar := progressbar.NewOptions(len(changedFiles),
|
||||||
progressbar.OptionSetDescription("Creating HDIFF files"),
|
progressbar.OptionSetDescription("📦 Creating HDiff files"),
|
||||||
progressbar.OptionShowCount(),
|
progressbar.OptionShowCount(),
|
||||||
progressbar.OptionSetWidth(30),
|
progressbar.OptionSetWidth(30),
|
||||||
progressbar.OptionSetPredictTime(true),
|
progressbar.OptionSetPredictTime(true),
|
||||||
@@ -54,7 +54,7 @@ func MakeHdiffFile(oldPath string, newPath string, changedFiles []string) error
|
|||||||
fmt.Fprintf(os.Stderr, "failed to create dir: %v\n", err)
|
fmt.Fprintf(os.Stderr, "failed to create dir: %v\n", err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
runHdiffz(oldFile, newFile, hdiffPath)
|
Hdiffz(oldFile, newFile, hdiffPath)
|
||||||
bar.Add(1)
|
bar.Add(1)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -4,11 +4,13 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"time"
|
||||||
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
"runtime"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"github.com/amenzhinsky/go-memexec"
|
||||||
"github.com/schollz/progressbar/v3"
|
"github.com/schollz/progressbar/v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -97,11 +99,6 @@ func ZipWith7za(src, dest string) error {
|
|||||||
return fmt.Errorf("source folder is empty: %s", src)
|
return fmt.Errorf("source folder is empty: %s", src)
|
||||||
}
|
}
|
||||||
|
|
||||||
sevenZipPath, err := filepath.Abs(filepath.Join("bin", "7za.exe"))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
destAbs, err := filepath.Abs(filepath.Join(".", dest))
|
destAbs, err := filepath.Abs(filepath.Join(".", dest))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -112,50 +109,75 @@ func ZipWith7za(src, dest string) error {
|
|||||||
args = append(args, f.Name())
|
args = append(args, f.Name())
|
||||||
}
|
}
|
||||||
|
|
||||||
cmd := exec.Command(sevenZipPath, args...)
|
exe, err := memexec.New(sevenZip)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer exe.Close()
|
||||||
|
cmd := exe.Command(args...)
|
||||||
cmd.Dir = src
|
cmd.Dir = src
|
||||||
cmd.Stdout = os.Stdout
|
cmd.Stdout = nil
|
||||||
cmd.Stderr = os.Stderr
|
cmd.Stderr = nil
|
||||||
|
|
||||||
return cmd.Run()
|
return cmd.Run()
|
||||||
}
|
}
|
||||||
|
|
||||||
func RemoveFolderWithProgress(folder string) error {
|
func RemoveFolderWithProgress(folder string, title string) error {
|
||||||
var total int
|
var total int
|
||||||
filepath.Walk(folder, func(path string, info os.FileInfo, err error) error {
|
filepath.Walk(folder, func(path string, info os.FileInfo, err error) error {
|
||||||
if err == nil {
|
if err == nil && !info.IsDir() {
|
||||||
total++
|
total++
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
bar := progressbar.NewOptions(total,
|
bar := progressbar.NewOptions(total,
|
||||||
progressbar.OptionSetDescription("Removing temp files"),
|
progressbar.OptionSetDescription(title),
|
||||||
progressbar.OptionShowCount(),
|
progressbar.OptionShowCount(),
|
||||||
progressbar.OptionSetWidth(30),
|
progressbar.OptionSetWidth(30),
|
||||||
progressbar.OptionSetPredictTime(true),
|
progressbar.OptionSetPredictTime(true),
|
||||||
)
|
)
|
||||||
|
|
||||||
err := filepath.Walk(folder, func(path string, info os.FileInfo, err error) error {
|
_ = filepath.Walk(folder, func(path string, info os.FileInfo, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return nil
|
||||||
}
|
}
|
||||||
if !info.IsDir() {
|
if info.IsDir() {
|
||||||
if err := os.Remove(path); err != nil {
|
return nil
|
||||||
return err
|
}
|
||||||
|
|
||||||
|
var rmErr error
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
rmErr = os.Remove(path)
|
||||||
|
if rmErr == nil {
|
||||||
|
break
|
||||||
}
|
}
|
||||||
bar.Add(1)
|
time.Sleep(300 * time.Millisecond)
|
||||||
}
|
}
|
||||||
|
if rmErr != nil {
|
||||||
|
fmt.Printf("⚠️ Could not remove %s: %v\n", path, rmErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
bar.Add(1)
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
|
if err := os.RemoveAll(folder); err != nil {
|
||||||
|
return fmt.Errorf("failed to remove folder %s: %w", folder, err)
|
||||||
|
}
|
||||||
|
bar.Finish()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func Hdiffz(oldPath, newPath, outDiff string) error {
|
||||||
|
args := []string{"-s-64", "-SD", "-c-zstd-21-24", "-d", oldPath, newPath, outDiff}
|
||||||
|
exe, err := memexec.New(hdiffz)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := os.RemoveAll(folder); err != nil {
|
defer exe.Close()
|
||||||
return err
|
cmd := exe.Command(args...)
|
||||||
}
|
cmd.Stdout = nil
|
||||||
|
cmd.Stderr = nil
|
||||||
|
|
||||||
bar.Finish()
|
return cmd.Run()
|
||||||
fmt.Println("\nTemp folder removed")
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
BIN
ldiff-converter/bin/7za.exe
Normal file
BIN
ldiff-converter/bin/7za.exe
Normal file
Binary file not shown.
@@ -3,7 +3,16 @@ module ldiff-converter
|
|||||||
go 1.25.0
|
go 1.25.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/amenzhinsky/go-memexec v0.7.1
|
||||||
github.com/klauspost/compress v1.18.0
|
github.com/klauspost/compress v1.18.0
|
||||||
golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b
|
golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b
|
||||||
google.golang.org/protobuf v1.36.8
|
google.golang.org/protobuf v1.36.8
|
||||||
)
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||||
|
github.com/rivo/uniseg v0.4.7 // indirect
|
||||||
|
github.com/schollz/progressbar/v3 v3.18.0 // indirect
|
||||||
|
golang.org/x/sys v0.29.0 // indirect
|
||||||
|
golang.org/x/term v0.28.0 // indirect
|
||||||
|
)
|
||||||
|
|||||||
@@ -1,6 +1,20 @@
|
|||||||
|
github.com/amenzhinsky/go-memexec v0.7.1 h1:DVm4cXzklaNWZoTJgZUi/dlXtelhC7QBtX4luKjl1qk=
|
||||||
|
github.com/amenzhinsky/go-memexec v0.7.1/go.mod h1:ApTO9/i2bcii7kvIXi74gum+/zYDzkiOXtuBZoYOKVE=
|
||||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||||
|
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
|
||||||
|
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
|
||||||
|
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||||
|
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||||
|
github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA=
|
||||||
|
github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
|
||||||
golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b h1:DXr+pvt3nC887026GRP39Ej11UATqWDmWuS99x26cD0=
|
golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b h1:DXr+pvt3nC887026GRP39Ej11UATqWDmWuS99x26cD0=
|
||||||
golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b/go.mod h1:4QTo5u+SEIbbKW1RacMZq1YEfOBqeXa19JeshGi+zc4=
|
golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b/go.mod h1:4QTo5u+SEIbbKW1RacMZq1YEfOBqeXa19JeshGi+zc4=
|
||||||
|
golang.org/x/sys v0.0.0-20210903071746-97244b99971b h1:3Dq0eVHn0uaQJmPO+/aYPI/fRMqdrVDbu7MQcku54gg=
|
||||||
|
golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
|
||||||
|
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
|
golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=
|
||||||
|
golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
|
||||||
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
|
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
|
||||||
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
|
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
|
||||||
|
|||||||
Binary file not shown.
@@ -1,15 +1,15 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"ldiff-converter/pb"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"ldiff-converter/pb"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
"golang.org/x/exp/mmap"
|
"golang.org/x/exp/mmap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
func LDiffFile(data *pb.AssetManifest, assetName string, assetSize int64, ldiffsDir, outputDir string) error {
|
func LDiffFile(data *pb.AssetManifest, assetName string, assetSize int64, ldiffsDir, outputDir string) error {
|
||||||
path := filepath.Join(ldiffsDir, data.ChunkFileName)
|
path := filepath.Join(ldiffsDir, data.ChunkFileName)
|
||||||
|
|
||||||
@@ -17,15 +17,25 @@ func LDiffFile(data *pb.AssetManifest, assetName string, assetSize int64, ldiffs
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("%s does not exist: %w", path, err)
|
return fmt.Errorf("%s does not exist: %w", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
fileSize := info.Size()
|
fileSize := info.Size()
|
||||||
|
|
||||||
var buffer []byte
|
var buffer []byte
|
||||||
|
|
||||||
if fileSize > 10*1024*1024 && data.HdiffFileSize > 1*1024*1024 {
|
if fileSize > 10*1024*1024 && data.HdiffFileSize > 1*1024*1024 {
|
||||||
// mmap for large files using x/exp/mmap
|
// ưu tiên mmap cho file lớn
|
||||||
reader, err := mmap.Open(path)
|
reader, err := mmap.Open(path)
|
||||||
if err != nil {
|
if err == nil {
|
||||||
// fallback to buffered read
|
defer reader.Close()
|
||||||
|
buffer = make([]byte, data.HdiffFileSize)
|
||||||
|
n, err := reader.ReadAt(buffer, data.HdiffFileInChunkOffset)
|
||||||
|
if err != nil && err != io.EOF {
|
||||||
|
return fmt.Errorf("error reading mmap data: %w", err)
|
||||||
|
}
|
||||||
|
if int64(n) < data.HdiffFileSize {
|
||||||
|
return fmt.Errorf("expected %d bytes, but read %d bytes", data.HdiffFileSize, n)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// fallback sang buffered read
|
||||||
file, err := os.Open(path)
|
file, err := os.Open(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("error opening file %s: %w", path, err)
|
return fmt.Errorf("error opening file %s: %w", path, err)
|
||||||
@@ -35,22 +45,14 @@ func LDiffFile(data *pb.AssetManifest, assetName string, assetSize int64, ldiffs
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
defer reader.Close()
|
|
||||||
buffer = make([]byte, data.HdiffFileSize)
|
|
||||||
_, err := reader.ReadAt(buffer, data.HdiffFileInChunkOffset)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("error reading mmap data: %w", err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// small files, buffered read
|
// file nhỏ, dùng buffered read
|
||||||
file, err := os.Open(path)
|
file, err := os.Open(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("error opening file %s: %w", path, err)
|
return fmt.Errorf("error opening file %s: %w", path, err)
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
|
|
||||||
buffer, err = ReadBuffer(file, data.HdiffFileInChunkOffset, data.HdiffFileSize)
|
buffer, err = ReadBuffer(file, data.HdiffFileInChunkOffset, data.HdiffFileSize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -58,7 +60,7 @@ func LDiffFile(data *pb.AssetManifest, assetName string, assetSize int64, ldiffs
|
|||||||
}
|
}
|
||||||
|
|
||||||
extension := ""
|
extension := ""
|
||||||
if data.OriginalFileSize != 0 || assetSize != data.HdiffFileSize {
|
if data.OriginalFileSize > 0 && assetSize != data.HdiffFileSize {
|
||||||
extension = ".hdiff"
|
extension = ".hdiff"
|
||||||
}
|
}
|
||||||
assetPath := filepath.Join(outputDir, assetName+extension)
|
assetPath := filepath.Join(outputDir, assetName+extension)
|
||||||
@@ -76,19 +78,16 @@ func LDiffFile(data *pb.AssetManifest, assetName string, assetSize int64, ldiffs
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
func ReadBuffer(file *os.File, offset int64, size int64) ([]byte, error) {
|
func ReadBuffer(file *os.File, offset int64, size int64) ([]byte, error) {
|
||||||
buffer := make([]byte, size)
|
buffer := make([]byte, size)
|
||||||
|
|
||||||
n, err := file.ReadAt(buffer, offset)
|
n, err := file.ReadAt(buffer, offset)
|
||||||
if err != nil {
|
if err != nil && err != io.EOF {
|
||||||
return nil, fmt.Errorf("error reading data: %w", err)
|
return nil, fmt.Errorf("error reading data: %w", err)
|
||||||
}
|
}
|
||||||
if int64(n) != size {
|
if int64(n) < size {
|
||||||
return nil, fmt.Errorf("expected %d bytes, but read %d bytes", size, n)
|
return nil, fmt.Errorf("expected %d bytes, but read %d bytes", size, n)
|
||||||
}
|
}
|
||||||
|
|
||||||
return buffer, nil
|
return buffer, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,18 +2,23 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
|
_ "embed"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"ldiff-converter/pb"
|
"ldiff-converter/pb"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/schollz/progressbar/v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
//go:embed bin/7za.exe
|
||||||
|
var sevenZip []byte
|
||||||
|
|
||||||
|
func main() {
|
||||||
reader := bufio.NewReader(os.Stdin)
|
reader := bufio.NewReader(os.Stdin)
|
||||||
fmt.Print("Enter zip ldiff path: ")
|
fmt.Print("Enter ldiff path: ")
|
||||||
ldiff, _ := reader.ReadString('\n')
|
ldiff, _ := reader.ReadString('\n')
|
||||||
ldiff = strings.TrimSpace(ldiff)
|
ldiff = strings.TrimSpace(ldiff)
|
||||||
if ldiff == "" {
|
if ldiff == "" {
|
||||||
@@ -28,6 +33,9 @@ func main() {
|
|||||||
fmt.Fprintln(os.Stderr, "no hdiff output provided")
|
fmt.Fprintln(os.Stderr, "no hdiff output provided")
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
if !strings.HasSuffix(strings.ToLower(hdiff), ".zip") {
|
||||||
|
hdiff += ".zip"
|
||||||
|
}
|
||||||
|
|
||||||
tmpFolderPath := filepath.Join(".", "temp")
|
tmpFolderPath := filepath.Join(".", "temp")
|
||||||
if err := os.MkdirAll(tmpFolderPath, 0755); err != nil {
|
if err := os.MkdirAll(tmpFolderPath, 0755); err != nil {
|
||||||
@@ -35,20 +43,24 @@ func main() {
|
|||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := Unzip(ldiff, tmpFolderPath); err != nil {
|
fmt.Println("Unzipping ldiff...")
|
||||||
|
if err := UnzipWith7za(ldiff, tmpFolderPath); err != nil {
|
||||||
fmt.Fprintln(os.Stderr, "error:", err)
|
fmt.Fprintln(os.Stderr, "error:", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
fmt.Println("Unzipping ldiff done.")
|
||||||
|
|
||||||
ldiffPath := filepath.Join(tmpFolderPath, "ldiff")
|
ldiffPath := filepath.Join(tmpFolderPath, "ldiff")
|
||||||
manifestPath := filepath.Join(tmpFolderPath, "manifest")
|
manifestPath := filepath.Join(tmpFolderPath, "manifest")
|
||||||
hdiffFolderPath := filepath.Join(".", "hdiff")
|
hdiffFolderPath := filepath.Join(".", "hdiff")
|
||||||
|
|
||||||
|
fmt.Println("Loading manifest proto...")
|
||||||
manifestProto, err := LoadManifestProto(manifestPath)
|
manifestProto, err := LoadManifestProto(manifestPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintln(os.Stderr, "error loading manifest proto:", err)
|
fmt.Fprintln(os.Stderr, "error loading manifest proto:", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
fmt.Println("Loading manifest proto done.")
|
||||||
|
|
||||||
ldiffEntries, err := os.ReadDir(ldiffPath)
|
ldiffEntries, err := os.ReadDir(ldiffPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -56,6 +68,13 @@ func main() {
|
|||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bar := progressbar.NewOptions(len(ldiffEntries),
|
||||||
|
progressbar.OptionSetDescription("📦 Converting ldiff files"),
|
||||||
|
progressbar.OptionShowCount(),
|
||||||
|
progressbar.OptionSetWidth(30),
|
||||||
|
progressbar.OptionSetPredictTime(true),
|
||||||
|
)
|
||||||
|
fmt.Println("Converting ldiff files...")
|
||||||
for _, ldiffEntry := range ldiffEntries {
|
for _, ldiffEntry := range ldiffEntries {
|
||||||
assetName := ldiffEntry.Name()
|
assetName := ldiffEntry.Name()
|
||||||
var matchingAssets []struct {
|
var matchingAssets []struct {
|
||||||
@@ -77,7 +96,7 @@ func main() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
bar.Add(1)
|
||||||
for _, ma := range matchingAssets {
|
for _, ma := range matchingAssets {
|
||||||
err := LDiffFile(ma.Asset, ma.AssetName, ma.AssetSize, ldiffPath, hdiffFolderPath)
|
err := LDiffFile(ma.Asset, ma.AssetName, ma.AssetSize, ldiffPath, hdiffFolderPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -85,12 +104,15 @@ func main() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
bar.Finish()
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println("Converting ldiff files done.")
|
||||||
diffMapNames := make([]string, len(ldiffEntries))
|
diffMapNames := make([]string, len(ldiffEntries))
|
||||||
for i, e := range ldiffEntries {
|
for i, e := range ldiffEntries {
|
||||||
diffMapNames[i] = e.Name()
|
diffMapNames[i] = e.Name()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fmt.Println("Making diff map...")
|
||||||
diffMapList, err := MakeDiffMap(manifestProto, diffMapNames)
|
diffMapList, err := MakeDiffMap(manifestProto, diffMapNames)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintln(os.Stderr, "error making diff map:", err)
|
fmt.Fprintln(os.Stderr, "error making diff map:", err)
|
||||||
@@ -107,22 +129,31 @@ func main() {
|
|||||||
fmt.Fprintln(os.Stderr, "error writing diff map:", err)
|
fmt.Fprintln(os.Stderr, "error writing diff map:", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
fmt.Println("Making diff map done.")
|
||||||
|
|
||||||
if err := os.RemoveAll(tmpFolderPath); err != nil {
|
fmt.Println("Removing temp ldiff files...")
|
||||||
|
if err := RemoveFolderWithProgress(tmpFolderPath, "🗑️ Removing temp ldiff files"); err != nil {
|
||||||
fmt.Fprintln(os.Stderr, "error removing temp dir:", err)
|
fmt.Fprintln(os.Stderr, "error removing temp dir:", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println("Removing temp ldiff files done.")
|
||||||
|
|
||||||
if err := Zip(hdiffFolderPath, hdiff); err != nil {
|
fmt.Println("Zipping hdiff files...")
|
||||||
|
if err := ZipWith7za(hdiffFolderPath, hdiff); err != nil {
|
||||||
fmt.Fprintln(os.Stderr, "error zip hdiff:", err)
|
fmt.Fprintln(os.Stderr, "error zip hdiff:", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
fmt.Println("Zipping hdiff files done.")
|
||||||
|
|
||||||
if err := os.RemoveAll(hdiffFolderPath); err != nil {
|
fmt.Println("Removing hdiff temp files...")
|
||||||
|
if err := RemoveFolderWithProgress(hdiffFolderPath, "🗑️ Removing hdiff temp files"); err != nil {
|
||||||
fmt.Fprintln(os.Stderr, "error removing temp dir:", err)
|
fmt.Fprintln(os.Stderr, "error removing temp dir:", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println("Removing hdiff temp files done.")
|
||||||
|
|
||||||
fmt.Println("done!")
|
fmt.Println("Done!")
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,112 +1,90 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"archive/zip"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"ldiff-converter/pb"
|
"ldiff-converter/pb"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"time"
|
||||||
|
|
||||||
|
"github.com/amenzhinsky/go-memexec"
|
||||||
|
"github.com/schollz/progressbar/v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
func Unzip(src, dest string) error {
|
func UnzipWith7za(src, dest string) error {
|
||||||
r, err := zip.OpenReader(src)
|
if _, err := os.Stat(src); os.IsNotExist(err) {
|
||||||
|
return fmt.Errorf("source file does not exist: %s", src)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.MkdirAll(dest, 0755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
destAbs, err := filepath.Abs(dest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer r.Close()
|
|
||||||
|
|
||||||
for _, f := range r.File {
|
exe, err := memexec.New(sevenZip)
|
||||||
fpath := filepath.Join(dest, f.Name)
|
if err != nil {
|
||||||
if !strings.HasPrefix(fpath, filepath.Clean(dest)+string(os.PathSeparator)) {
|
return err
|
||||||
return fmt.Errorf("illegal file path: %s", fpath)
|
}
|
||||||
}
|
defer exe.Close()
|
||||||
|
|
||||||
if f.FileInfo().IsDir() {
|
args := []string{"x", "-y", "-o" + destAbs, src}
|
||||||
if err := os.MkdirAll(fpath, f.Mode()); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := os.MkdirAll(filepath.Dir(fpath), 0755); err != nil {
|
cmd := exe.Command(args...)
|
||||||
return err
|
cmd.Stdout = nil
|
||||||
}
|
cmd.Stderr = nil
|
||||||
|
if err := cmd.Run(); err != nil {
|
||||||
rc, err := f.Open()
|
return err
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
|
|
||||||
if err != nil {
|
|
||||||
rc.Close()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = io.Copy(outFile, rc)
|
|
||||||
rc.Close()
|
|
||||||
outFile.Close()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func Zip(src, dest string) error {
|
func ZipWith7za(src, dest string) error {
|
||||||
zipFile, err := os.Create(dest)
|
if _, err := os.Stat(src); os.IsNotExist(err) {
|
||||||
|
return fmt.Errorf("source folder does not exist: %s", src)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
files, err := os.ReadDir(src)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer zipFile.Close()
|
|
||||||
|
|
||||||
zw := zip.NewWriter(zipFile)
|
if len(files) == 0 {
|
||||||
defer zw.Close()
|
return fmt.Errorf("source folder is empty: %s", src)
|
||||||
|
}
|
||||||
|
|
||||||
err = filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
|
destAbs, err := filepath.Abs(filepath.Join(".", dest))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
relPath, err := filepath.Rel(src, path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if info.IsDir() {
|
|
||||||
if relPath == "." {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
_, err := zw.Create(relPath + "/")
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
fh, err := zip.FileInfoHeader(info)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fh.Name = relPath
|
|
||||||
fh.Method = zip.Deflate
|
|
||||||
|
|
||||||
writer, err := zw.CreateHeader(fh)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
file, err := os.Open(path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer file.Close()
|
|
||||||
|
|
||||||
_, err = io.Copy(writer, file)
|
|
||||||
return err
|
return err
|
||||||
})
|
}
|
||||||
|
|
||||||
return err
|
args := []string{"a", "-tzip", "-mx=1", "-mmt=on", destAbs}
|
||||||
|
for _, f := range files {
|
||||||
|
args = append(args, f.Name())
|
||||||
|
}
|
||||||
|
|
||||||
|
exe, err := memexec.New(sevenZip)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer exe.Close()
|
||||||
|
cmd := exe.Command(args...)
|
||||||
|
cmd.Dir = src
|
||||||
|
cmd.Stdout = nil
|
||||||
|
cmd.Stderr = nil
|
||||||
|
if err := cmd.Run(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func MakeDiffMap(manifest *pb.ManifestProto, chunkNames []string) ([]*HDiffData, error) {
|
func MakeDiffMap(manifest *pb.ManifestProto, chunkNames []string) ([]*HDiffData, error) {
|
||||||
var hdiffFiles []*HDiffData
|
var hdiffFiles []*HDiffData
|
||||||
|
|
||||||
@@ -140,3 +118,51 @@ func MakeDiffMap(manifest *pb.ManifestProto, chunkNames []string) ([]*HDiffData,
|
|||||||
|
|
||||||
return hdiffFiles, nil
|
return hdiffFiles, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func RemoveFolderWithProgress(folder string, title string) error {
|
||||||
|
var total int
|
||||||
|
filepath.Walk(folder, func(path string, info os.FileInfo, err error) error {
|
||||||
|
if err == nil && !info.IsDir() {
|
||||||
|
total++
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
bar := progressbar.NewOptions(total,
|
||||||
|
progressbar.OptionSetDescription(title),
|
||||||
|
progressbar.OptionShowCount(),
|
||||||
|
progressbar.OptionSetWidth(30),
|
||||||
|
progressbar.OptionSetPredictTime(true),
|
||||||
|
)
|
||||||
|
|
||||||
|
_ = filepath.Walk(folder, func(path string, info os.FileInfo, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if info.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Retry tối đa 10 lần khi xóa
|
||||||
|
var rmErr error
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
rmErr = os.Remove(path)
|
||||||
|
if rmErr == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(300 * time.Millisecond)
|
||||||
|
}
|
||||||
|
if rmErr != nil {
|
||||||
|
fmt.Printf("⚠️ Could not remove %s: %v\n", path, rmErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
bar.Add(1)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
if err := os.RemoveAll(folder); err != nil {
|
||||||
|
return fmt.Errorf("failed to remove folder %s: %w", folder, err)
|
||||||
|
}
|
||||||
|
bar.Finish()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user