=Start=
缘由:
在学习Go 语言的过程中整理总结一下常见功能的实现代码,方便以后需要的时候参考。
正文:
参考解答:
话不多说,先上代码。先可以运行,再逐步按需理解和改进。
package main import ( "archive/tar" "compress/gzip" "flag" "fmt" "io" "log" "os" "path" ) // main functions shows how to TarGz a directory and // UnTarGz a file func main() { var srcDirPath = *flag.String("srcDir", "testdata", "The source directory name.") var targetFilePath = *flag.String("targetName", "testdata.tar.gz", "The target file name.") flag.Parse() fmt.Printf("srcDirPath: %s, type: %T\n", srcDirPath, srcDirPath) fmt.Printf("targetFilePath: %s, type: %T\n", targetFilePath, targetFilePath) TarGz(srcDirPath, targetFilePath) // 压缩 UnTarGz(targetFilePath, srcDirPath+"_temp") // 解压 } // Gzip and tar from source directory or file to destination file // you need check file exist before you call this function func TarGz(srcDirPath string, destFilePath string) { fw, err := os.Create(destFilePath) handleError(err) defer fw.Close() // Gzip writer gw := gzip.NewWriter(fw) defer gw.Close() // Tar writer tw := tar.NewWriter(gw) defer tw.Close() // Check if it's a file or a directory f, err := os.Open(srcDirPath) handleError(err) fi, err := f.Stat() handleError(err) if fi.IsDir() { // handle source directory fmt.Println("Cerating tar.gz from directory...") tarGzDir(srcDirPath, path.Base(srcDirPath), tw) } else { // handle file directly fmt.Println("Cerating tar.gz from " + fi.Name() + "...") tarGzFile(srcDirPath, fi.Name(), tw, fi) } fmt.Println("Well done!") } // Deal with directories // if find files, handle them with tarGzFile // Every recurrence append the base path to the recPath // recPath is the path inside of tar.gz func tarGzDir(srcDirPath string, recPath string, tw *tar.Writer) { // Open source diretory dir, err := os.Open(srcDirPath) handleError(err) defer dir.Close() // Get file info slice fis, err := dir.Readdir(0) handleError(err) for _, fi := range fis { // Append path curPath := srcDirPath + "/" + fi.Name() // Check it is directory or file if fi.IsDir() { // Directory // (Directory won't add unitl all subfiles are added) fmt.Printf("Adding path...%s\\n", curPath) tarGzDir(curPath, recPath+"/"+fi.Name(), tw) } else { // File fmt.Printf("Adding file...%s\\n", curPath) } tarGzFile(curPath, recPath+"/"+fi.Name(), tw, fi) } } // Deal with files func tarGzFile(srcFile string, recPath string, tw *tar.Writer, fi os.FileInfo) { if fi.IsDir() { // Create tar header hdr := new(tar.Header) // if last character of header name is '/' it also can be directory // but if you don't set Typeflag, error will occur when you untargz hdr.Name = recPath + "/" hdr.Typeflag = tar.TypeDir hdr.Size = 0 //hdr.Mode = 0755 | c_ISDIR hdr.Mode = int64(fi.Mode()) hdr.ModTime = fi.ModTime() // Write hander err := tw.WriteHeader(hdr) handleError(err) } else { // File reader fr, err := os.Open(srcFile) handleError(err) defer fr.Close() // Create tar header hdr := new(tar.Header) hdr.Name = recPath hdr.Size = fi.Size() hdr.Mode = int64(fi.Mode()) hdr.ModTime = fi.ModTime() // Write hander err = tw.WriteHeader(hdr) handleError(err) // Write file data _, err = io.Copy(tw, fr) handleError(err) } } // Ungzip and untar from source file to destination directory // you need check file exist before you call this function func UnTarGz(srcFilePath string, destDirPath string) { fmt.Println("UnTarGzing " + srcFilePath + "...") // Create destination directory os.Mkdir(destDirPath, os.ModePerm) fr, err := os.Open(srcFilePath) handleError(err) defer fr.Close() // Gzip reader gr, err := gzip.NewReader(fr) // Tar reader tr := tar.NewReader(gr) for { hdr, err := tr.Next() if err == io.EOF { // End of tar archive break } //handleError(err) fmt.Println("UnTarGzing file..." + hdr.Name) // Check if it is diretory or file if hdr.Typeflag != tar.TypeDir { // Get files from archive // Create diretory before create file os.MkdirAll(destDirPath+"/"+path.Dir(hdr.Name), os.ModePerm) // Write data to file fw, _ := os.Create(destDirPath + "/" + hdr.Name) handleError(err) _, err = io.Copy(fw, tr) handleError(err) } } fmt.Println("Well done!") } func handleError(err error) { log.Println(err) }
&
package main import ( "archive/tar" "compress/gzip" "flag" "fmt" "io" "os" "path/filepath" "strings" ) func main() { var srcDirPath = *flag.String("srcDir", "testdata", "The source directory name.") var targetFilePath = *flag.String("targetName", "testdata.tar.gz", "The target file name.") flag.Parse() fmt.Printf("srcDirPath: %s, type: %T\n", srcDirPath, srcDirPath) fmt.Printf("targetFilePath: %s, type: %T\n", targetFilePath, targetFilePath) TarFilesDirs(srcDirPath, targetFilePath) // 仅压缩,解压可以使用tar命令进行 } func TarFilesDirs(path string, tarFilePath string) error { file, err := os.Create(tarFilePath) if err != nil { return err } defer file.Close() gz := gzip.NewWriter(file) defer gz.Close() tw := tar.NewWriter(gz) defer tw.Close() if err := tarit(path, tw); err != nil { return err } return nil } func tarit(source string, tw *tar.Writer) error { info, err := os.Stat(source) if err != nil { return nil } var baseDir string if info.IsDir() { baseDir = filepath.Base(source) } return filepath.Walk(source, func(path string, info os.FileInfo, err error) error { if err != nil { return err } var link string if info.Mode()&os.ModeSymlink != 0 { if link, err = os.Readlink(path); err != nil { return err } } header, err := tar.FileInfoHeader(info, link) if err != nil { return err } if baseDir != "" { header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source)) } if !info.Mode().IsRegular() { //nothing more to do for non-regular return nil } if err := tw.WriteHeader(header); err != nil { return err } if info.IsDir() { return nil } file, err := os.Open(path) if err != nil { return err } defer file.Close() buf := make([]byte, 16) if _, err = io.CopyBuffer(tw, file, buf); err != nil { return err } return nil }) }
参考链接:
- Quick golang script for creating a gzipped tarball#测试OK,但比较简单
- Go语言实现tar.gz打包解包#测试OK,简单加工之后可使用
- Golang 实现tar.gz打包#测试OK,且比较简洁
- 代码片段 – Golang 创建 .tar.gz 压缩包
- Golang-压缩-解压-Tar.Gz
- golang压缩,解压tar.gz文件
- golang tar gzip 压缩,解压(含目录文件)
- https://golang.org/pkg/archive/tar/
- Golang: Working with Gzip and Tar
https://gist.github.com/svett/76799ba2edca89961be6 - Golang 学习笔记(四)- archive/tar 实现打包压缩及解压
https://broqiang.com/posts/45 - 5.压缩文件读写
=END=
《 “Go语言学习#14-实现tar.gz格式的压缩解压” 》 有 3 条评论
【Go】使用压缩文件优化io (一)
https://blog.thinkeridea.com/201906/go/compress_file_io_optimization1.html
`
最近遇到一个日志备份 io 过高的问题,业务日志每十分钟备份一次,本来是用 Python 写一个根据规则扫描备份日志问题不大,但是随着业务越来越多,单机上的日志文件越来越大,文件数量也越来越多,导致每每备份的瞬间 io 阻塞严重, CPU 和 load 异常的高,好在备份速度很快,对业务影响不是很大,这个问题会随着业务增长,越来越明显,这段时间抽空对备份方式做了优化,效果十分显著,整理篇文章记录一下。
`
【Go】使用压缩文件优化io (二)
https://blog.thinkeridea.com/201907/go/compress_file_io_optimization2.html
https://www.reddit.com/r/golang/comments/2z8u63/access_to_java_jar_file_information_from_go/
easy way to unzip file with golang
https://stackoverflow.com/questions/20357223/easy-way-to-unzip-file-with-golang
https://github.com/mholt/archiver
https://github.com/mholt/archiver/blob/master/zip.go
Unzip Files in Go
https://golangcode.com/unzip-files-in-go/
https://golangcode.com/create-zip-files-in-go/
Go实战–压缩zip和解压缩unzip的应用(The way to go)
https://blog.csdn.net/wangshubo1989/article/details/71743374
How to list the files inside a JAR file?
https://stackoverflow.com/questions/1429172/how-to-list-the-files-inside-a-jar-file
Extract only a specific file from a zipped archive to a given directory
https://unix.stackexchange.com/questions/14120/extract-only-a-specific-file-from-a-zipped-archive-to-a-given-directory