diff --git a/README.md b/README.md index 7a975de..18abbb0 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,4 @@ # Some-Tool-For-Diff +## HDiff any game +## LDiff converter diff --git a/hdiff-any-game/Makefile b/hdiff-any-game/Makefile new file mode 100644 index 0000000..5e93e77 --- /dev/null +++ b/hdiff-any-game/Makefile @@ -0,0 +1,12 @@ +APP_NAME = hdiff-any-game.exe + +all: build + +build: + go build -o $(APP_NAME) ./ + +run: build + ./$(APP_NAME) + +clean: + rm -f $(APP_NAME) diff.txt \ No newline at end of file diff --git a/hdiff-any-game/bin/7za.exe b/hdiff-any-game/bin/7za.exe new file mode 100644 index 0000000..b322bf8 Binary files /dev/null and b/hdiff-any-game/bin/7za.exe differ diff --git a/hdiff-any-game/bin/hdiffz.exe b/hdiff-any-game/bin/hdiffz.exe new file mode 100644 index 0000000..9607bd0 Binary files /dev/null and b/hdiff-any-game/bin/hdiffz.exe differ diff --git a/hdiff-any-game/checker.go b/hdiff-any-game/checker.go new file mode 100644 index 0000000..9843322 --- /dev/null +++ b/hdiff-any-game/checker.go @@ -0,0 +1,138 @@ +package main + +import ( + "crypto/md5" + "encoding/binary" + "encoding/hex" + "io" + "os" + "runtime" + "sync" + + "github.com/schollz/progressbar/v3" +) + +type DiffResult struct { + OnlyInOld []string + OnlyInNew []string + Changed []string +} + +func safePartialMD5(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + fi, err := f.Stat() + if err != nil { + return "", err + } + size := fi.Size() + modTime := fi.ModTime().UnixNano() + + h := md5.New() + + binary.Write(h, binary.LittleEndian, size) + binary.Write(h, binary.LittleEndian, modTime) + binary.Write(h, binary.LittleEndian, fi.Mode()) + + buf := make([]byte, 4096) + + if size <= 16*1024 { + f.Seek(0, io.SeekStart) + io.Copy(h, f) + return hex.EncodeToString(h.Sum(nil)), nil + } + + n, _ := f.Read(buf) + h.Write(buf[:n]) + + if size > 8192 { + mid := size / 2 + f.Seek(mid-2048, io.SeekStart) + n, _ = f.Read(buf) + h.Write(buf[:n]) + } + + if size > 4096 { + f.Seek(-4096, io.SeekEnd) + n, _ = f.Read(buf) + h.Write(buf[:n]) + } + + return hex.EncodeToString(h.Sum(nil)), nil +} + +func DiffFolders(oldPath, newPath string) (*DiffResult, error) { + oldFiles, err := collectFiles(oldPath) + if err != nil { + return nil, err + } + newFiles, err := collectFiles(newPath) + if err != nil { + return nil, err + } + + result := &DiffResult{} + var mu sync.Mutex + var wg sync.WaitGroup + + jobs := make(chan [3]string, 100) + + total := 0 + for rel := range oldFiles { + if _, ok := newFiles[rel]; ok { + total++ + } + } + + bar := progressbar.NewOptions(total, + progressbar.OptionSetDescription("๐Ÿ” Comparing files"), + progressbar.OptionSetWidth(30), + progressbar.OptionShowCount(), + progressbar.OptionSetPredictTime(true), + progressbar.OptionShowElapsedTimeOnFinish(), + ) + + workers := runtime.NumCPU() * 2 + + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for job := range jobs { + rel, oldFile, newFile := job[0], job[1], job[2] + oldHash, _ := safePartialMD5(oldFile) + newHash, _ := safePartialMD5(newFile) + if oldHash != newHash { + mu.Lock() + result.Changed = append(result.Changed, rel) + mu.Unlock() + } + bar.Add(1) + } + }() + } + + for rel, oldFile := range oldFiles { + if newFile, ok := newFiles[rel]; ok { + jobs <- [3]string{rel, oldFile, newFile} + } else { + result.OnlyInOld = append(result.OnlyInOld, rel) + } + } + + for rel := range newFiles { + if _, ok := oldFiles[rel]; !ok { + result.OnlyInNew = append(result.OnlyInNew, rel) + } + } + + close(jobs) + wg.Wait() + bar.Finish() + + return result, nil +} diff --git a/hdiff-any-game/copyNewFIle.go b/hdiff-any-game/copyNewFIle.go new file mode 100644 index 0000000..a528b91 --- /dev/null +++ b/hdiff-any-game/copyNewFIle.go @@ -0,0 +1,41 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/schollz/progressbar/v3" +) + +func CopyNewFiles(newPath string, result *DiffResult) error { + delFile, err := os.Create("hdiff/deletefiles.txt") + if err != nil { + return err + } + defer delFile.Close() + for _, f := range result.OnlyInOld { + fmt.Fprintln(delFile, f) + } + + bar := progressbar.NewOptions(len(result.OnlyInNew), + progressbar.OptionSetDescription("๐Ÿ“‚ Copying new files"), + progressbar.OptionSetWidth(30), + progressbar.OptionShowCount(), + progressbar.OptionSetPredictTime(true), + ) + + for _, rel := range result.OnlyInNew { + src := filepath.Join(newPath, rel) + dst := filepath.Join("hdiff", rel) + os.MkdirAll(filepath.Dir(dst), 0755) + + if err := copyFile(src, dst); err != nil { + fmt.Println("copy error:", err) + } + bar.Add(1) + } + bar.Finish() + + return nil +} diff --git a/hdiff-any-game/go.mod b/hdiff-any-game/go.mod new file mode 100644 index 0000000..398057a --- /dev/null +++ b/hdiff-any-game/go.mod @@ -0,0 +1,11 @@ +module hdiff-any-game + +go 1.25.0 + +require ( + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/schollz/progressbar/v3 v3.18.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/term v0.34.0 // indirect +) diff --git a/hdiff-any-game/go.sum b/hdiff-any-game/go.sum new file mode 100644 index 0000000..9eab304 --- /dev/null +++ b/hdiff-any-game/go.sum @@ -0,0 +1,10 @@ +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA= +github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= diff --git a/hdiff-any-game/hdiff-any-game.exe b/hdiff-any-game/hdiff-any-game.exe new file mode 100644 index 0000000..2139230 Binary files /dev/null and b/hdiff-any-game/hdiff-any-game.exe differ diff --git a/hdiff-any-game/hdiffz.go b/hdiff-any-game/hdiffz.go new file mode 100644 index 0000000..311dd2c --- /dev/null +++ b/hdiff-any-game/hdiffz.go @@ -0,0 +1,19 @@ +package main + +import ( + "os/exec" + "path/filepath" +) + +func runHdiffz(oldPath, newPath, outDiff string) error { + args := []string{"-s-64", "-SD", "-c-zstd-21-24", "-d", oldPath, newPath, outDiff} + hdiffzPath, err := filepath.Abs(filepath.Join("bin", "hdiffz.exe")) + if err != nil { + return err + } + cmd := exec.Command(hdiffzPath, args...) + cmd.Stdout = nil + cmd.Stderr = nil + + return cmd.Run() +} diff --git a/hdiff-any-game/main.go b/hdiff-any-game/main.go new file mode 100644 index 0000000..d1ae831 --- /dev/null +++ b/hdiff-any-game/main.go @@ -0,0 +1,112 @@ +package main + +import ( + "bufio" + "embed" + "fmt" + "os" + "path/filepath" + "strings" +) + +//go:embed bin/hdiffz.exe +//go:embed bin/7za.exe +var embeddedFiles embed.FS + +func ensureBinaries() (map[string]string, error) { + binDir := "bin" + if _, err := os.Stat(binDir); os.IsNotExist(err) { + if err := os.MkdirAll(binDir, 0755); err != nil { + return nil, err + } + } + + files := []string{"hdiffz.exe", "7za.exe"} + paths := make(map[string]string) + + for _, f := range files { + destPath := filepath.Join(binDir, f) + paths[f] = destPath + + if _, err := os.Stat(destPath); os.IsNotExist(err) { + data, err := embeddedFiles.ReadFile("bin/" + f) + if err != nil { + return nil, err + } + if err := os.WriteFile(destPath, data, 0755); err != nil { + return nil, err + } + } + } + + return paths, nil +} + +func main() { + paths, err := ensureBinaries() + if err != nil { + fmt.Println("Error:", err) + return + } + + for _, path := range paths { + fmt.Println("Binary ready at:", path) + } + reader := bufio.NewReader(os.Stdin) + + fmt.Print("Enter OLD game path: ") + oldPath, _ := reader.ReadString('\n') + oldPath = strings.TrimSpace(oldPath) + if oldPath == "" { + fmt.Fprintln(os.Stderr, "no old path provided") + os.Exit(1) + } + + fmt.Print("Enter NEW game path: ") + newPath, _ := reader.ReadString('\n') + newPath = strings.TrimSpace(newPath) + if newPath == "" { + fmt.Fprintln(os.Stderr, "no new path provided") + os.Exit(1) + } + + fmt.Print("Enter zip hdiff output name: ") + hdiffName, _ := reader.ReadString('\n') + hdiffName = strings.TrimSpace(hdiffName) + if hdiffName == "" { + fmt.Fprintln(os.Stderr, "no hdiff output provided") + os.Exit(1) + } + + if !strings.HasSuffix(strings.ToLower(hdiffName), ".zip") { + hdiffName += ".zip" + } + + result, err := DiffFolders(oldPath, newPath) + if err != nil { + fmt.Println("Error:", err) + return + } + hdiffFolderPath := filepath.Join(".", "hdiff") + os.MkdirAll(hdiffFolderPath, 0755) + if err := CopyNewFiles(newPath, result); err != nil { + fmt.Println("Error writing diff:", err) + return + } + if err := MakeHdiffFile(oldPath, newPath, result.Changed); err != nil { + fmt.Println("Error writing diff:", err) + return + } + + if err := ZipWith7za(hdiffFolderPath, hdiffName); err != nil { + fmt.Println("Error writing diff:", err) + return + } + + if err := RemoveFolderWithProgress(hdiffFolderPath); err != nil { + fmt.Fprintln(os.Stderr, "error removing temp dir:", err) + os.Exit(1) + } + + fmt.Println("Done") +} diff --git a/hdiff-any-game/makeHdiffFile.go b/hdiff-any-game/makeHdiffFile.go new file mode 100644 index 0000000..f68db90 --- /dev/null +++ b/hdiff-any-game/makeHdiffFile.go @@ -0,0 +1,71 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "runtime" + "sync" + + "github.com/schollz/progressbar/v3" +) + +type HdiffFile struct { + RemoteName string `json:"remoteName"` +} + +func MakeHdiffFile(oldPath string, newPath string, changedFiles []string) error { + delFile, err := os.Create("hdiff/hdifffiles.txt") + if err != nil { + return err + } + defer delFile.Close() + + for _, f := range changedFiles { + data, err := json.Marshal(HdiffFile{RemoteName: f}) + if err != nil { + return err + } + fmt.Fprintln(delFile, string(data)) + } + + bar := progressbar.NewOptions(len(changedFiles), + progressbar.OptionSetDescription("Creating HDIFF files"), + progressbar.OptionShowCount(), + progressbar.OptionSetWidth(30), + progressbar.OptionSetPredictTime(true), + ) + + workers := runtime.NumCPU() / 2 + if workers < 2 { + workers = 2 + } + jobs := make(chan string, len(changedFiles)) + var wg sync.WaitGroup + + for i := int64(0); i < int64(workers); i++ { + wg.Go(func() { + for f := range jobs { + oldFile := filepath.Join(oldPath, f) + newFile := filepath.Join(newPath, f) + hdiffPath := filepath.Join("hdiff", f+".hdiff") + if err := os.MkdirAll(filepath.Dir(hdiffPath), 0755); err != nil { + fmt.Fprintf(os.Stderr, "failed to create dir: %v\n", err) + continue + } + runHdiffz(oldFile, newFile, hdiffPath) + bar.Add(1) + } + }) + } + + for _, f := range changedFiles { + jobs <- f + } + close(jobs) + + wg.Wait() + bar.Finish() + return nil +} diff --git a/hdiff-any-game/utils.go b/hdiff-any-game/utils.go new file mode 100644 index 0000000..18eb6a4 --- /dev/null +++ b/hdiff-any-game/utils.go @@ -0,0 +1,161 @@ +package main + +import ( + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "sync" + + "github.com/schollz/progressbar/v3" +) + +func collectFiles(root string) (map[string]string, error) { + files := sync.Map{} + var wg sync.WaitGroup + dirs := make(chan string, 100) + + workers := runtime.NumCPU() * 2 + if workers < 1 { + workers = 1 + } + + for i := 0; i < workers; i++ { + go func() { + for dir := range dirs { + entries, err := os.ReadDir(dir) + if err != nil { + wg.Done() + continue + } + for _, e := range entries { + path := filepath.Join(dir, e.Name()) + if e.IsDir() { + wg.Add(1) + dirs <- path + } else { + rel, _ := filepath.Rel(root, path) + files.Store(filepath.ToSlash(rel), path) + } + } + wg.Done() + } + }() + } + wg.Add(1) + dirs <- root + + go func() { + wg.Wait() + close(dirs) + }() + + wg.Wait() + out := make(map[string]string) + files.Range(func(k, v any) bool { + out[k.(string)] = v.(string) + return true + }) + + return out, nil +} + +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, in) + return err +} + +func ZipWith7za(src, dest string) error { + if _, err := os.Stat(src); os.IsNotExist(err) { + return fmt.Errorf("source folder does not exist: %s", src) + } + + if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil { + return err + } + + files, err := os.ReadDir(src) + if err != nil { + return err + } + + if len(files) == 0 { + return fmt.Errorf("source folder is empty: %s", src) + } + + sevenZipPath, err := filepath.Abs(filepath.Join("bin", "7za.exe")) + if err != nil { + return err + } + + destAbs, err := filepath.Abs(filepath.Join(".", dest)) + if err != nil { + return err + } + + args := []string{"a", "-tzip", "-mx=1", "-mmt=on", destAbs} + for _, f := range files { + args = append(args, f.Name()) + } + + cmd := exec.Command(sevenZipPath, args...) + cmd.Dir = src + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + return cmd.Run() +} + +func RemoveFolderWithProgress(folder string) error { + var total int + filepath.Walk(folder, func(path string, info os.FileInfo, err error) error { + if err == nil { + total++ + } + return nil + }) + + bar := progressbar.NewOptions(total, + progressbar.OptionSetDescription("Removing temp files"), + progressbar.OptionShowCount(), + progressbar.OptionSetWidth(30), + progressbar.OptionSetPredictTime(true), + ) + + err := filepath.Walk(folder, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + if err := os.Remove(path); err != nil { + return err + } + bar.Add(1) + } + return nil + }) + if err != nil { + return err + } + if err := os.RemoveAll(folder); err != nil { + return err + } + + bar.Finish() + fmt.Println("\nTemp folder removed") + return nil +} diff --git a/ldiff-converter/Makefile b/ldiff-converter/Makefile new file mode 100644 index 0000000..3ce1215 --- /dev/null +++ b/ldiff-converter/Makefile @@ -0,0 +1,12 @@ +APP_NAME = ldff-converter.exe + +all: build + +build: + go build -o $(APP_NAME) ./ + +run: build + ./$(APP_NAME) + +clean: + rm -f $(APP_NAME) diff --git a/ldiff-converter/go.mod b/ldiff-converter/go.mod new file mode 100644 index 0000000..fbee9d0 --- /dev/null +++ b/ldiff-converter/go.mod @@ -0,0 +1,9 @@ +module ldiff-converter + +go 1.25.0 + +require ( + github.com/klauspost/compress v1.18.0 + golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b + google.golang.org/protobuf v1.36.8 +) diff --git a/ldiff-converter/go.sum b/ldiff-converter/go.sum new file mode 100644 index 0000000..62c0902 --- /dev/null +++ b/ldiff-converter/go.sum @@ -0,0 +1,6 @@ +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b h1:DXr+pvt3nC887026GRP39Ej11UATqWDmWuS99x26cD0= +golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b/go.mod h1:4QTo5u+SEIbbKW1RacMZq1YEfOBqeXa19JeshGi+zc4= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= diff --git a/ldiff-converter/ldff-converter.exe b/ldiff-converter/ldff-converter.exe new file mode 100644 index 0000000..9a2c701 Binary files /dev/null and b/ldiff-converter/ldff-converter.exe differ diff --git a/ldiff-converter/ldiff.go b/ldiff-converter/ldiff.go new file mode 100644 index 0000000..3af7054 --- /dev/null +++ b/ldiff-converter/ldiff.go @@ -0,0 +1,94 @@ +package main + +import ( + "ldiff-converter/pb" + "fmt" + "os" + "path/filepath" + + "golang.org/x/exp/mmap" +) + + +func LDiffFile(data *pb.AssetManifest, assetName string, assetSize int64, ldiffsDir, outputDir string) error { + path := filepath.Join(ldiffsDir, data.ChunkFileName) + + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("%s does not exist: %w", path, err) + } + + fileSize := info.Size() + var buffer []byte + + if fileSize > 10*1024*1024 && data.HdiffFileSize > 1*1024*1024 { + // mmap for large files using x/exp/mmap + reader, err := mmap.Open(path) + if err != nil { + // fallback to buffered read + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("error opening file %s: %w", path, err) + } + defer file.Close() + buffer, err = ReadBuffer(file, data.HdiffFileInChunkOffset, data.HdiffFileSize) + if err != nil { + return err + } + } else { + defer reader.Close() + buffer = make([]byte, data.HdiffFileSize) + _, err := reader.ReadAt(buffer, data.HdiffFileInChunkOffset) + if err != nil { + return fmt.Errorf("error reading mmap data: %w", err) + } + } + } else { + // small files, buffered read + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("error opening file %s: %w", path, err) + } + defer file.Close() + + buffer, err = ReadBuffer(file, data.HdiffFileInChunkOffset, data.HdiffFileSize) + if err != nil { + return err + } + } + + extension := "" + if data.OriginalFileSize != 0 || assetSize != data.HdiffFileSize { + extension = ".hdiff" + } + assetPath := filepath.Join(outputDir, assetName+extension) + parentDir := filepath.Dir(assetPath) + if _, err := os.Stat(parentDir); os.IsNotExist(err) { + if err := os.MkdirAll(parentDir, 0o755); err != nil { + return fmt.Errorf("error creating directory %s: %w", parentDir, err) + } + } + + if err := os.WriteFile(assetPath, buffer, 0o644); err != nil { + return fmt.Errorf("error writing file %s: %w", assetPath, err) + } + + return nil +} + + + +func ReadBuffer(file *os.File, offset int64, size int64) ([]byte, error) { + buffer := make([]byte, size) + + n, err := file.ReadAt(buffer, offset) + if err != nil { + return nil, fmt.Errorf("error reading data: %w", err) + } + if int64(n) != size { + return nil, fmt.Errorf("expected %d bytes, but read %d bytes", size, n) + } + + return buffer, nil +} + diff --git a/ldiff-converter/loader.go b/ldiff-converter/loader.go new file mode 100644 index 0000000..071c158 --- /dev/null +++ b/ldiff-converter/loader.go @@ -0,0 +1,66 @@ +package main + +import ( + "bufio" + "ldiff-converter/pb" + "io" + "os" + + "github.com/klauspost/compress/zstd" + "google.golang.org/protobuf/proto" +) + +func LoadChunkProto(path string) (*pb.ChunkProto, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + + reader := bufio.NewReader(file) + decoder, err := zstd.NewReader(reader) + if err != nil { + return nil, err + } + defer decoder.Close() + + data, err := io.ReadAll(decoder) + if err != nil { + return nil, err + } + + var chunk pb.ChunkProto + if err := proto.Unmarshal(data, &chunk); err != nil { + return nil, err + } + + return &chunk, nil +} + +// Load ManifestProto tแปซ file Zstd + Protobuf +func LoadManifestProto(path string) (*pb.ManifestProto, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + + reader := bufio.NewReader(file) + decoder, err := zstd.NewReader(reader) + if err != nil { + return nil, err + } + defer decoder.Close() + + data, err := io.ReadAll(decoder) + if err != nil { + return nil, err + } + + var manifest pb.ManifestProto + if err := proto.Unmarshal(data, &manifest); err != nil { + return nil, err + } + + return &manifest, nil +} \ No newline at end of file diff --git a/ldiff-converter/main.go b/ldiff-converter/main.go new file mode 100644 index 0000000..e44d99a --- /dev/null +++ b/ldiff-converter/main.go @@ -0,0 +1,128 @@ +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "ldiff-converter/pb" + "os" + "path/filepath" + "strings" +) + +func main() { + + reader := bufio.NewReader(os.Stdin) + fmt.Print("Enter zip ldiff path: ") + ldiff, _ := reader.ReadString('\n') + ldiff = strings.TrimSpace(ldiff) + if ldiff == "" { + fmt.Fprintln(os.Stderr, "no ldiff file provided") + os.Exit(1) + } + + fmt.Print("Enter zip hdiff output: ") + hdiff, _ := reader.ReadString('\n') + hdiff = strings.TrimSpace(hdiff) + if hdiff == "" { + fmt.Fprintln(os.Stderr, "no hdiff output provided") + os.Exit(1) + } + + tmpFolderPath := filepath.Join(".", "temp") + if err := os.MkdirAll(tmpFolderPath, 0755); err != nil { + fmt.Fprintln(os.Stderr, "error creating temp dir:", err) + os.Exit(1) + } + + if err := Unzip(ldiff, tmpFolderPath); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } + + ldiffPath := filepath.Join(tmpFolderPath, "ldiff") + manifestPath := filepath.Join(tmpFolderPath, "manifest") + hdiffFolderPath := filepath.Join(".", "hdiff") + + manifestProto, err := LoadManifestProto(manifestPath) + if err != nil { + fmt.Fprintln(os.Stderr, "error loading manifest proto:", err) + os.Exit(1) + } + + ldiffEntries, err := os.ReadDir(ldiffPath) + if err != nil { + fmt.Fprintln(os.Stderr, "error reading ldiff dir:", err) + os.Exit(1) + } + + for _, ldiffEntry := range ldiffEntries { + assetName := ldiffEntry.Name() + var matchingAssets []struct { + AssetName string + AssetSize int64 + Asset *pb.AssetManifest + } + + for _, assetGroup := range manifestProto.Assets { + if data := assetGroup.AssetData; data != nil { + for _, asset := range data.Assets { + if asset.ChunkFileName == assetName { + matchingAssets = append(matchingAssets, struct { + AssetName string + AssetSize int64 + Asset *pb.AssetManifest + }{assetGroup.AssetName, assetGroup.AssetSize, asset}) + } + } + } + } + + for _, ma := range matchingAssets { + err := LDiffFile(ma.Asset, ma.AssetName, ma.AssetSize, ldiffPath, hdiffFolderPath) + if err != nil { + continue + } + } + } + + diffMapNames := make([]string, len(ldiffEntries)) + for i, e := range ldiffEntries { + diffMapNames[i] = e.Name() + } + + diffMapList, err := MakeDiffMap(manifestProto, diffMapNames) + if err != nil { + fmt.Fprintln(os.Stderr, "error making diff map:", err) + os.Exit(1) + } + diffMapJson, err := json.Marshal(diffMapList) + if err != nil { + fmt.Fprintln(os.Stderr, "error marshal diff map:", err) + os.Exit(1) + } + + diffMapJsonPath := filepath.Join(hdiffFolderPath, "hdifffiles.json") + if err := os.WriteFile(diffMapJsonPath, diffMapJson, 0644); err != nil { + fmt.Fprintln(os.Stderr, "error writing diff map:", err) + os.Exit(1) + } + + if err := os.RemoveAll(tmpFolderPath); err != nil { + fmt.Fprintln(os.Stderr, "error removing temp dir:", err) + os.Exit(1) + } + + if err := Zip(hdiffFolderPath, hdiff); err != nil { + fmt.Fprintln(os.Stderr, "error zip hdiff:", err) + os.Exit(1) + } + + if err := os.RemoveAll(hdiffFolderPath); err != nil { + fmt.Fprintln(os.Stderr, "error removing temp dir:", err) + os.Exit(1) + } + + fmt.Println("done!") + +} diff --git a/ldiff-converter/model.go b/ldiff-converter/model.go new file mode 100644 index 0000000..57fba3f --- /dev/null +++ b/ldiff-converter/model.go @@ -0,0 +1,7 @@ +package main + +type HDiffData struct { + SourceFileName string `json:"source_file_name"` + TargetFileName string `json:"target_file_name"` + PatchFileName string `json:"patch_file_name"` +} \ No newline at end of file diff --git a/ldiff-converter/pb/firefly.pb.go b/ldiff-converter/pb/firefly.pb.go new file mode 100644 index 0000000..9543a86 --- /dev/null +++ b/ldiff-converter/pb/firefly.pb.go @@ -0,0 +1,651 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.5 +// protoc v4.25.6 +// source: proto/firefly.proto + +package pb + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ChunkProto struct { + state protoimpl.MessageState `protogen:"open.v1"` + Assets []*AssetChunkProperty `protobuf:"bytes,1,rep,name=assets,proto3" json:"assets,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ChunkProto) Reset() { + *x = ChunkProto{} + mi := &file_proto_firefly_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ChunkProto) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChunkProto) ProtoMessage() {} + +func (x *ChunkProto) ProtoReflect() protoreflect.Message { + mi := &file_proto_firefly_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChunkProto.ProtoReflect.Descriptor instead. +func (*ChunkProto) Descriptor() ([]byte, []int) { + return file_proto_firefly_proto_rawDescGZIP(), []int{0} +} + +func (x *ChunkProto) GetAssets() []*AssetChunkProperty { + if x != nil { + return x.Assets + } + return nil +} + +type AssetChunkProperty struct { + state protoimpl.MessageState `protogen:"open.v1"` + AssetName string `protobuf:"bytes,1,opt,name=asset_name,json=assetName,proto3" json:"asset_name,omitempty"` + AssetChunks []*AssetChunk `protobuf:"bytes,2,rep,name=asset_chunks,json=assetChunks,proto3" json:"asset_chunks,omitempty"` + AssetType int32 `protobuf:"varint,3,opt,name=asset_type,json=assetType,proto3" json:"asset_type,omitempty"` + AssetSize int64 `protobuf:"varint,4,opt,name=asset_size,json=assetSize,proto3" json:"asset_size,omitempty"` + AssetHashMd5 string `protobuf:"bytes,5,opt,name=asset_hash_md5,json=assetHashMd5,proto3" json:"asset_hash_md5,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AssetChunkProperty) Reset() { + *x = AssetChunkProperty{} + mi := &file_proto_firefly_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AssetChunkProperty) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AssetChunkProperty) ProtoMessage() {} + +func (x *AssetChunkProperty) ProtoReflect() protoreflect.Message { + mi := &file_proto_firefly_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AssetChunkProperty.ProtoReflect.Descriptor instead. +func (*AssetChunkProperty) Descriptor() ([]byte, []int) { + return file_proto_firefly_proto_rawDescGZIP(), []int{1} +} + +func (x *AssetChunkProperty) GetAssetName() string { + if x != nil { + return x.AssetName + } + return "" +} + +func (x *AssetChunkProperty) GetAssetChunks() []*AssetChunk { + if x != nil { + return x.AssetChunks + } + return nil +} + +func (x *AssetChunkProperty) GetAssetType() int32 { + if x != nil { + return x.AssetType + } + return 0 +} + +func (x *AssetChunkProperty) GetAssetSize() int64 { + if x != nil { + return x.AssetSize + } + return 0 +} + +func (x *AssetChunkProperty) GetAssetHashMd5() string { + if x != nil { + return x.AssetHashMd5 + } + return "" +} + +type AssetChunk struct { + state protoimpl.MessageState `protogen:"open.v1"` + ChunkName string `protobuf:"bytes,1,opt,name=chunk_name,json=chunkName,proto3" json:"chunk_name,omitempty"` + ChunkDecompressedHashMd5 string `protobuf:"bytes,2,opt,name=chunk_decompressed_hash_md5,json=chunkDecompressedHashMd5,proto3" json:"chunk_decompressed_hash_md5,omitempty"` + ChunkOnFileOffset int64 `protobuf:"varint,3,opt,name=chunk_on_file_offset,json=chunkOnFileOffset,proto3" json:"chunk_on_file_offset,omitempty"` + ChunkSize int64 `protobuf:"varint,4,opt,name=chunk_size,json=chunkSize,proto3" json:"chunk_size,omitempty"` + ChunkSizeDecompressed int64 `protobuf:"varint,5,opt,name=chunk_size_decompressed,json=chunkSizeDecompressed,proto3" json:"chunk_size_decompressed,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AssetChunk) Reset() { + *x = AssetChunk{} + mi := &file_proto_firefly_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AssetChunk) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AssetChunk) ProtoMessage() {} + +func (x *AssetChunk) ProtoReflect() protoreflect.Message { + mi := &file_proto_firefly_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AssetChunk.ProtoReflect.Descriptor instead. +func (*AssetChunk) Descriptor() ([]byte, []int) { + return file_proto_firefly_proto_rawDescGZIP(), []int{2} +} + +func (x *AssetChunk) GetChunkName() string { + if x != nil { + return x.ChunkName + } + return "" +} + +func (x *AssetChunk) GetChunkDecompressedHashMd5() string { + if x != nil { + return x.ChunkDecompressedHashMd5 + } + return "" +} + +func (x *AssetChunk) GetChunkOnFileOffset() int64 { + if x != nil { + return x.ChunkOnFileOffset + } + return 0 +} + +func (x *AssetChunk) GetChunkSize() int64 { + if x != nil { + return x.ChunkSize + } + return 0 +} + +func (x *AssetChunk) GetChunkSizeDecompressed() int64 { + if x != nil { + return x.ChunkSizeDecompressed + } + return 0 +} + +type ManifestProto struct { + state protoimpl.MessageState `protogen:"open.v1"` + Assets []*AssetManifestProperty `protobuf:"bytes,1,rep,name=assets,proto3" json:"assets,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ManifestProto) Reset() { + *x = ManifestProto{} + mi := &file_proto_firefly_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ManifestProto) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ManifestProto) ProtoMessage() {} + +func (x *ManifestProto) ProtoReflect() protoreflect.Message { + mi := &file_proto_firefly_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ManifestProto.ProtoReflect.Descriptor instead. +func (*ManifestProto) Descriptor() ([]byte, []int) { + return file_proto_firefly_proto_rawDescGZIP(), []int{3} +} + +func (x *ManifestProto) GetAssets() []*AssetManifestProperty { + if x != nil { + return x.Assets + } + return nil +} + +type AssetManifestProperty struct { + state protoimpl.MessageState `protogen:"open.v1"` + AssetName string `protobuf:"bytes,1,opt,name=asset_name,json=assetName,proto3" json:"asset_name,omitempty"` + AssetSize int64 `protobuf:"varint,2,opt,name=asset_size,json=assetSize,proto3" json:"asset_size,omitempty"` + AssetHashMd5 string `protobuf:"bytes,3,opt,name=asset_hash_md5,json=assetHashMd5,proto3" json:"asset_hash_md5,omitempty"` + AssetData *AssetManifestChunk `protobuf:"bytes,4,opt,name=asset_data,json=assetData,proto3" json:"asset_data,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AssetManifestProperty) Reset() { + *x = AssetManifestProperty{} + mi := &file_proto_firefly_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AssetManifestProperty) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AssetManifestProperty) ProtoMessage() {} + +func (x *AssetManifestProperty) ProtoReflect() protoreflect.Message { + mi := &file_proto_firefly_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AssetManifestProperty.ProtoReflect.Descriptor instead. +func (*AssetManifestProperty) Descriptor() ([]byte, []int) { + return file_proto_firefly_proto_rawDescGZIP(), []int{4} +} + +func (x *AssetManifestProperty) GetAssetName() string { + if x != nil { + return x.AssetName + } + return "" +} + +func (x *AssetManifestProperty) GetAssetSize() int64 { + if x != nil { + return x.AssetSize + } + return 0 +} + +func (x *AssetManifestProperty) GetAssetHashMd5() string { + if x != nil { + return x.AssetHashMd5 + } + return "" +} + +func (x *AssetManifestProperty) GetAssetData() *AssetManifestChunk { + if x != nil { + return x.AssetData + } + return nil +} + +type AssetManifestChunk struct { + state protoimpl.MessageState `protogen:"open.v1"` + LatestAssetVersion string `protobuf:"bytes,1,opt,name=latest_asset_version,json=latestAssetVersion,proto3" json:"latest_asset_version,omitempty"` + Assets []*AssetManifest `protobuf:"bytes,2,rep,name=assets,proto3" json:"assets,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AssetManifestChunk) Reset() { + *x = AssetManifestChunk{} + mi := &file_proto_firefly_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AssetManifestChunk) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AssetManifestChunk) ProtoMessage() {} + +func (x *AssetManifestChunk) ProtoReflect() protoreflect.Message { + mi := &file_proto_firefly_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AssetManifestChunk.ProtoReflect.Descriptor instead. +func (*AssetManifestChunk) Descriptor() ([]byte, []int) { + return file_proto_firefly_proto_rawDescGZIP(), []int{5} +} + +func (x *AssetManifestChunk) GetLatestAssetVersion() string { + if x != nil { + return x.LatestAssetVersion + } + return "" +} + +func (x *AssetManifestChunk) GetAssets() []*AssetManifest { + if x != nil { + return x.Assets + } + return nil +} + +type AssetManifest struct { + state protoimpl.MessageState `protogen:"open.v1"` + ChunkFileName string `protobuf:"bytes,1,opt,name=chunk_file_name,json=chunkFileName,proto3" json:"chunk_file_name,omitempty"` + ChunkFileVersion string `protobuf:"bytes,2,opt,name=chunk_file_version,json=chunkFileVersion,proto3" json:"chunk_file_version,omitempty"` + ChunkFileNode string `protobuf:"bytes,3,opt,name=chunk_file_node,json=chunkFileNode,proto3" json:"chunk_file_node,omitempty"` + ChunkFileSize int64 `protobuf:"varint,4,opt,name=chunk_file_size,json=chunkFileSize,proto3" json:"chunk_file_size,omitempty"` + ChunkFileMd5 string `protobuf:"bytes,5,opt,name=chunk_file_md5,json=chunkFileMd5,proto3" json:"chunk_file_md5,omitempty"` + HdiffFileInChunkOffset int64 `protobuf:"varint,6,opt,name=hdiff_file_in_chunk_offset,json=hdiffFileInChunkOffset,proto3" json:"hdiff_file_in_chunk_offset,omitempty"` + HdiffFileSize int64 `protobuf:"varint,7,opt,name=hdiff_file_size,json=hdiffFileSize,proto3" json:"hdiff_file_size,omitempty"` + OriginalFilePath string `protobuf:"bytes,8,opt,name=original_file_path,json=originalFilePath,proto3" json:"original_file_path,omitempty"` + OriginalFileSize int64 `protobuf:"varint,9,opt,name=original_file_size,json=originalFileSize,proto3" json:"original_file_size,omitempty"` + OriginalFileMd5 string `protobuf:"bytes,10,opt,name=original_file_md5,json=originalFileMd5,proto3" json:"original_file_md5,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AssetManifest) Reset() { + *x = AssetManifest{} + mi := &file_proto_firefly_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AssetManifest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AssetManifest) ProtoMessage() {} + +func (x *AssetManifest) ProtoReflect() protoreflect.Message { + mi := &file_proto_firefly_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AssetManifest.ProtoReflect.Descriptor instead. +func (*AssetManifest) Descriptor() ([]byte, []int) { + return file_proto_firefly_proto_rawDescGZIP(), []int{6} +} + +func (x *AssetManifest) GetChunkFileName() string { + if x != nil { + return x.ChunkFileName + } + return "" +} + +func (x *AssetManifest) GetChunkFileVersion() string { + if x != nil { + return x.ChunkFileVersion + } + return "" +} + +func (x *AssetManifest) GetChunkFileNode() string { + if x != nil { + return x.ChunkFileNode + } + return "" +} + +func (x *AssetManifest) GetChunkFileSize() int64 { + if x != nil { + return x.ChunkFileSize + } + return 0 +} + +func (x *AssetManifest) GetChunkFileMd5() string { + if x != nil { + return x.ChunkFileMd5 + } + return "" +} + +func (x *AssetManifest) GetHdiffFileInChunkOffset() int64 { + if x != nil { + return x.HdiffFileInChunkOffset + } + return 0 +} + +func (x *AssetManifest) GetHdiffFileSize() int64 { + if x != nil { + return x.HdiffFileSize + } + return 0 +} + +func (x *AssetManifest) GetOriginalFilePath() string { + if x != nil { + return x.OriginalFilePath + } + return "" +} + +func (x *AssetManifest) GetOriginalFileSize() int64 { + if x != nil { + return x.OriginalFileSize + } + return 0 +} + +func (x *AssetManifest) GetOriginalFileMd5() string { + if x != nil { + return x.OriginalFileMd5 + } + return "" +} + +var File_proto_firefly_proto protoreflect.FileDescriptor + +var file_proto_firefly_proto_rawDesc = string([]byte{ + 0x0a, 0x13, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x66, 0x69, 0x72, 0x65, 0x66, 0x6c, 0x79, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x07, 0x66, 0x69, 0x72, 0x65, 0x66, 0x6c, 0x79, 0x22, 0x41, + 0x0a, 0x0a, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x33, 0x0a, 0x06, + 0x61, 0x73, 0x73, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x66, + 0x69, 0x72, 0x65, 0x66, 0x6c, 0x79, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x74, 0x43, 0x68, 0x75, 0x6e, + 0x6b, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x52, 0x06, 0x61, 0x73, 0x73, 0x65, 0x74, + 0x73, 0x22, 0xcf, 0x01, 0x0a, 0x12, 0x41, 0x73, 0x73, 0x65, 0x74, 0x43, 0x68, 0x75, 0x6e, 0x6b, + 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x73, 0x73, 0x65, + 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x73, + 0x73, 0x65, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x36, 0x0a, 0x0c, 0x61, 0x73, 0x73, 0x65, 0x74, + 0x5f, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, + 0x66, 0x69, 0x72, 0x65, 0x66, 0x6c, 0x79, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x74, 0x43, 0x68, 0x75, + 0x6e, 0x6b, 0x52, 0x0b, 0x61, 0x73, 0x73, 0x65, 0x74, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x12, + 0x1d, 0x0a, 0x0a, 0x61, 0x73, 0x73, 0x65, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x09, 0x61, 0x73, 0x73, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, + 0x0a, 0x0a, 0x61, 0x73, 0x73, 0x65, 0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x09, 0x61, 0x73, 0x73, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x24, 0x0a, + 0x0e, 0x61, 0x73, 0x73, 0x65, 0x74, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x5f, 0x6d, 0x64, 0x35, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x61, 0x73, 0x73, 0x65, 0x74, 0x48, 0x61, 0x73, 0x68, + 0x4d, 0x64, 0x35, 0x22, 0xf2, 0x01, 0x0a, 0x0a, 0x41, 0x73, 0x73, 0x65, 0x74, 0x43, 0x68, 0x75, + 0x6e, 0x6b, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x4e, 0x61, 0x6d, + 0x65, 0x12, 0x3d, 0x0a, 0x1b, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x64, 0x65, 0x63, 0x6f, 0x6d, + 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x5f, 0x6d, 0x64, 0x35, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x44, 0x65, 0x63, + 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x48, 0x61, 0x73, 0x68, 0x4d, 0x64, 0x35, + 0x12, 0x2f, 0x0a, 0x14, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x6f, 0x6e, 0x5f, 0x66, 0x69, 0x6c, + 0x65, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x11, + 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x4f, 0x6e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x66, 0x66, 0x73, 0x65, + 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x53, 0x69, 0x7a, 0x65, + 0x12, 0x36, 0x0a, 0x17, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x64, + 0x65, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x15, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x53, 0x69, 0x7a, 0x65, 0x44, 0x65, 0x63, 0x6f, + 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x22, 0x47, 0x0a, 0x0d, 0x4d, 0x61, 0x6e, 0x69, + 0x66, 0x65, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x36, 0x0a, 0x06, 0x61, 0x73, 0x73, + 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x66, 0x69, 0x72, 0x65, + 0x66, 0x6c, 0x79, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x74, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, + 0x74, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x52, 0x06, 0x61, 0x73, 0x73, 0x65, 0x74, + 0x73, 0x22, 0xb7, 0x01, 0x0a, 0x15, 0x41, 0x73, 0x73, 0x65, 0x74, 0x4d, 0x61, 0x6e, 0x69, 0x66, + 0x65, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x61, + 0x73, 0x73, 0x65, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x61, 0x73, 0x73, 0x65, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x73, + 0x73, 0x65, 0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, + 0x61, 0x73, 0x73, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x61, 0x73, 0x73, + 0x65, 0x74, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x5f, 0x6d, 0x64, 0x35, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0c, 0x61, 0x73, 0x73, 0x65, 0x74, 0x48, 0x61, 0x73, 0x68, 0x4d, 0x64, 0x35, 0x12, + 0x3a, 0x0a, 0x0a, 0x61, 0x73, 0x73, 0x65, 0x74, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x66, 0x69, 0x72, 0x65, 0x66, 0x6c, 0x79, 0x2e, 0x41, 0x73, + 0x73, 0x65, 0x74, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x43, 0x68, 0x75, 0x6e, 0x6b, + 0x52, 0x09, 0x61, 0x73, 0x73, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x22, 0x76, 0x0a, 0x12, 0x41, + 0x73, 0x73, 0x65, 0x74, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x43, 0x68, 0x75, 0x6e, + 0x6b, 0x12, 0x30, 0x0a, 0x14, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x61, 0x73, 0x73, 0x65, + 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x12, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x41, 0x73, 0x73, 0x65, 0x74, 0x56, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x2e, 0x0a, 0x06, 0x61, 0x73, 0x73, 0x65, 0x74, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x69, 0x72, 0x65, 0x66, 0x6c, 0x79, 0x2e, 0x41, 0x73, + 0x73, 0x65, 0x74, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x52, 0x06, 0x61, 0x73, 0x73, + 0x65, 0x74, 0x73, 0x22, 0xc7, 0x03, 0x0a, 0x0d, 0x41, 0x73, 0x73, 0x65, 0x74, 0x4d, 0x61, 0x6e, + 0x69, 0x66, 0x65, 0x73, 0x74, 0x12, 0x26, 0x0a, 0x0f, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x66, + 0x69, 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, + 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2c, 0x0a, + 0x12, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x68, 0x75, 0x6e, 0x6b, + 0x46, 0x69, 0x6c, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x26, 0x0a, 0x0f, 0x63, + 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x46, 0x69, 0x6c, 0x65, 0x4e, + 0x6f, 0x64, 0x65, 0x12, 0x26, 0x0a, 0x0f, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x66, 0x69, 0x6c, + 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x63, 0x68, + 0x75, 0x6e, 0x6b, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x63, + 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6d, 0x64, 0x35, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0c, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x46, 0x69, 0x6c, 0x65, 0x4d, 0x64, + 0x35, 0x12, 0x3a, 0x0a, 0x1a, 0x68, 0x64, 0x69, 0x66, 0x66, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, + 0x69, 0x6e, 0x5f, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x16, 0x68, 0x64, 0x69, 0x66, 0x66, 0x46, 0x69, 0x6c, 0x65, + 0x49, 0x6e, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x26, 0x0a, + 0x0f, 0x68, 0x64, 0x69, 0x66, 0x66, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x68, 0x64, 0x69, 0x66, 0x66, 0x46, 0x69, 0x6c, + 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x2c, 0x0a, 0x12, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, + 0x6c, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x08, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x10, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x46, 0x69, 0x6c, 0x65, 0x50, + 0x61, 0x74, 0x68, 0x12, 0x2c, 0x0a, 0x12, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x5f, + 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x10, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x69, 0x7a, + 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x5f, 0x66, 0x69, + 0x6c, 0x65, 0x5f, 0x6d, 0x64, 0x35, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x6f, 0x72, + 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x46, 0x69, 0x6c, 0x65, 0x4d, 0x64, 0x35, 0x42, 0x0e, 0x5a, + 0x0c, 0x2e, 0x2f, 0x66, 0x69, 0x72, 0x65, 0x66, 0x6c, 0x79, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, +}) + +var ( + file_proto_firefly_proto_rawDescOnce sync.Once + file_proto_firefly_proto_rawDescData []byte +) + +func file_proto_firefly_proto_rawDescGZIP() []byte { + file_proto_firefly_proto_rawDescOnce.Do(func() { + file_proto_firefly_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_proto_firefly_proto_rawDesc), len(file_proto_firefly_proto_rawDesc))) + }) + return file_proto_firefly_proto_rawDescData +} + +var file_proto_firefly_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_proto_firefly_proto_goTypes = []any{ + (*ChunkProto)(nil), // 0: firefly.ChunkProto + (*AssetChunkProperty)(nil), // 1: firefly.AssetChunkProperty + (*AssetChunk)(nil), // 2: firefly.AssetChunk + (*ManifestProto)(nil), // 3: firefly.ManifestProto + (*AssetManifestProperty)(nil), // 4: firefly.AssetManifestProperty + (*AssetManifestChunk)(nil), // 5: firefly.AssetManifestChunk + (*AssetManifest)(nil), // 6: firefly.AssetManifest +} +var file_proto_firefly_proto_depIdxs = []int32{ + 1, // 0: firefly.ChunkProto.assets:type_name -> firefly.AssetChunkProperty + 2, // 1: firefly.AssetChunkProperty.asset_chunks:type_name -> firefly.AssetChunk + 4, // 2: firefly.ManifestProto.assets:type_name -> firefly.AssetManifestProperty + 5, // 3: firefly.AssetManifestProperty.asset_data:type_name -> firefly.AssetManifestChunk + 6, // 4: firefly.AssetManifestChunk.assets:type_name -> firefly.AssetManifest + 5, // [5:5] is the sub-list for method output_type + 5, // [5:5] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name +} + +func init() { file_proto_firefly_proto_init() } +func file_proto_firefly_proto_init() { + if File_proto_firefly_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_proto_firefly_proto_rawDesc), len(file_proto_firefly_proto_rawDesc)), + NumEnums: 0, + NumMessages: 7, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_proto_firefly_proto_goTypes, + DependencyIndexes: file_proto_firefly_proto_depIdxs, + MessageInfos: file_proto_firefly_proto_msgTypes, + }.Build() + File_proto_firefly_proto = out.File + file_proto_firefly_proto_goTypes = nil + file_proto_firefly_proto_depIdxs = nil +} diff --git a/ldiff-converter/utils.go b/ldiff-converter/utils.go new file mode 100644 index 0000000..17273ca --- /dev/null +++ b/ldiff-converter/utils.go @@ -0,0 +1,142 @@ +package main + +import ( + "archive/zip" + "fmt" + "io" + "ldiff-converter/pb" + "os" + "path/filepath" + "strings" +) + +func Unzip(src, dest string) error { + r, err := zip.OpenReader(src) + if err != nil { + return err + } + defer r.Close() + + for _, f := range r.File { + fpath := filepath.Join(dest, f.Name) + if !strings.HasPrefix(fpath, filepath.Clean(dest)+string(os.PathSeparator)) { + return fmt.Errorf("illegal file path: %s", fpath) + } + + if f.FileInfo().IsDir() { + if err := os.MkdirAll(fpath, f.Mode()); err != nil { + return err + } + continue + } + + if err := os.MkdirAll(filepath.Dir(fpath), 0755); err != nil { + return err + } + + rc, err := f.Open() + if err != nil { + return err + } + + outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + rc.Close() + return err + } + + _, err = io.Copy(outFile, rc) + rc.Close() + outFile.Close() + if err != nil { + return err + } + } + return nil +} + +func Zip(src, dest string) error { + zipFile, err := os.Create(dest) + if err != nil { + return err + } + defer zipFile.Close() + + zw := zip.NewWriter(zipFile) + defer zw.Close() + + err = filepath.Walk(src, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + relPath, err := filepath.Rel(src, path) + if err != nil { + return err + } + + if info.IsDir() { + if relPath == "." { + return nil + } + _, err := zw.Create(relPath + "/") + return err + } + + fh, err := zip.FileInfoHeader(info) + if err != nil { + return err + } + fh.Name = relPath + fh.Method = zip.Deflate + + writer, err := zw.CreateHeader(fh) + if err != nil { + return err + } + + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + _, err = io.Copy(writer, file) + return err + }) + + return err +} +func MakeDiffMap(manifest *pb.ManifestProto, chunkNames []string) ([]*HDiffData, error) { + var hdiffFiles []*HDiffData + + for _, asset := range manifest.Assets { + assetName := asset.AssetName + assetSize := asset.AssetSize + + if asset.AssetData != nil { + for _, chunk := range asset.AssetData.Assets { + matched := false + for _, name := range chunkNames { + if name == chunk.ChunkFileName { + matched = true + break + } + } + if !matched { + continue + } + + if chunk.OriginalFileSize != 0 || chunk.HdiffFileSize != assetSize { + hdiffFiles = append(hdiffFiles, &HDiffData{ + SourceFileName: chunk.OriginalFilePath, + TargetFileName: assetName, + PatchFileName: fmt.Sprintf("%s.hdiff", assetName), + }) + } + } + } + } + + return hdiffFiles, nil +}