389 lines
12 KiB
Go
389 lines
12 KiB
Go
// Package archive walks and re-encodes archive files (zip / tar /
|
||
// tar.gz / tar.bz2 / tar.xz / 7z / rar) fully in-process — no native
|
||
// deps. It is shared by the file browser's extract/compress endpoints
|
||
// (internal/dispatch) and the direct update provider
|
||
// (internal/updater), which extracts archive downloads into the
|
||
// instance instead of dumping the raw archive bytes.
|
||
//
|
||
// 7z and rar use pure-Go libraries (bodgit/sevenzip,
|
||
// nwaples/rardecode/v2). Encrypted rars and solid 7z volumes are
|
||
// surfaced as errors, not silently skipped.
|
||
package archive
|
||
|
||
import (
|
||
"archive/tar"
|
||
"archive/zip"
|
||
"bytes"
|
||
"compress/bzip2"
|
||
"compress/gzip"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"path"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/bodgit/sevenzip"
|
||
"github.com/nwaples/rardecode/v2"
|
||
"github.com/ulikunitz/xz"
|
||
)
|
||
|
||
// Entry-count / byte caps so a malformed or hostile archive can't pin
|
||
// the agent forever walking phantom records.
|
||
const (
|
||
MaxEntries = 200000
|
||
MaxTotalBytes = 8 * 1024 * 1024 * 1024
|
||
EntrySoftBytes = 4 * 1024 * 1024 * 1024
|
||
)
|
||
|
||
// EntryFunc is called once per archive entry. body is a streaming
|
||
// reader (nil for directories); the callback may read fully or skip.
|
||
type EntryFunc func(name string, mode int64, isDir bool, body io.Reader) error
|
||
|
||
// Walk opens `data` as an archive (auto-detected via signature plus
|
||
// filename hint) and calls fn for each entry. Returns entry and byte
|
||
// counts alongside any error.
|
||
func Walk(data []byte, name string, fn EntryFunc) (int64, int64, error) {
|
||
switch DetectFormat(data, name) {
|
||
case "zip":
|
||
return walkZip(data, fn)
|
||
case "tar.gz":
|
||
gz, err := gzip.NewReader(bytes.NewReader(data))
|
||
if err != nil {
|
||
return 0, 0, fmt.Errorf("gzip open: %w", err)
|
||
}
|
||
defer gz.Close()
|
||
return walkTar(gz, fn)
|
||
case "tar.bz2":
|
||
return walkTar(bzip2.NewReader(bytes.NewReader(data)), fn)
|
||
case "tar.xz":
|
||
xr, err := xz.NewReader(bytes.NewReader(data))
|
||
if err != nil {
|
||
return 0, 0, fmt.Errorf("xz open: %w", err)
|
||
}
|
||
return walkTar(xr, fn)
|
||
case "tar":
|
||
return walkTar(bytes.NewReader(data), fn)
|
||
case "7z":
|
||
return walk7z(data, fn)
|
||
case "rar":
|
||
return walkRar(data, fn)
|
||
default:
|
||
return 0, 0, fmt.Errorf("unsupported archive format for %q (panel knows zip / tar / tar.gz / tar.bz2 / tar.xz / 7z / rar)", name)
|
||
}
|
||
}
|
||
|
||
// WriteAsTar parses an archive from `data` and re-emits its entries as
|
||
// a tar stream into `out`. Used by container-side extract paths so the
|
||
// agent can feed Docker's CopyToContainer in a single API call.
|
||
func WriteAsTar(data []byte, originalName string, out io.Writer) (int64, int64, error) {
|
||
tw := tar.NewWriter(out)
|
||
defer tw.Close()
|
||
var entries, bytesOut int64
|
||
_, _, err := Walk(data, originalName, func(name string, mode int64, isDir bool, body io.Reader) error {
|
||
safe := SafeEntryPath(name)
|
||
if safe == "" {
|
||
return nil
|
||
}
|
||
// Buffer the entry body so we can compute its length before
|
||
// writing the tar header. zip / gzip readers return streaming
|
||
// readers; the tar writer needs Size up front.
|
||
var bodyBuf bytes.Buffer
|
||
var sz int64
|
||
if !isDir {
|
||
n, err := io.Copy(&bodyBuf, body)
|
||
if err != nil {
|
||
return fmt.Errorf("entry %q: %w", name, err)
|
||
}
|
||
sz = n
|
||
if sz > EntrySoftBytes {
|
||
return fmt.Errorf("entry %q exceeds %d-byte cap", name, int64(EntrySoftBytes))
|
||
}
|
||
}
|
||
hdr := &tar.Header{Name: safe, ModTime: time.Now(), Mode: 0o644}
|
||
if mode != 0 {
|
||
hdr.Mode = mode & 0o777
|
||
}
|
||
if isDir {
|
||
hdr.Typeflag = tar.TypeDir
|
||
hdr.Mode = 0o755
|
||
if !strings.HasSuffix(hdr.Name, "/") {
|
||
hdr.Name += "/"
|
||
}
|
||
} else {
|
||
hdr.Typeflag = tar.TypeReg
|
||
hdr.Size = sz
|
||
}
|
||
if err := tw.WriteHeader(hdr); err != nil {
|
||
return err
|
||
}
|
||
if !isDir {
|
||
if _, err := tw.Write(bodyBuf.Bytes()); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
entries++
|
||
bytesOut += sz
|
||
return nil
|
||
})
|
||
if err != nil {
|
||
return entries, bytesOut, err
|
||
}
|
||
return entries, bytesOut, tw.Close()
|
||
}
|
||
|
||
// DetectFormat classifies `data` by magic bytes first (handles renamed
|
||
// extensions), then falls back to the filename hint. Returns "" when
|
||
// the content is not a recognized archive.
|
||
func DetectFormat(data []byte, name string) string {
|
||
lower := strings.ToLower(name)
|
||
// Magic bytes win when present — handles renamed extensions.
|
||
if len(data) >= 4 && data[0] == 0x50 && data[1] == 0x4b && (data[2] == 0x03 || data[2] == 0x05 || data[2] == 0x07) {
|
||
return "zip"
|
||
}
|
||
if len(data) >= 3 && data[0] == 0x1f && data[1] == 0x8b && data[2] == 0x08 {
|
||
return "tar.gz"
|
||
}
|
||
if len(data) >= 3 && data[0] == 'B' && data[1] == 'Z' && data[2] == 'h' {
|
||
return "tar.bz2"
|
||
}
|
||
// 7z signature: 37 7A BC AF 27 1C
|
||
if len(data) >= 6 && data[0] == 0x37 && data[1] == 0x7a && data[2] == 0xbc && data[3] == 0xaf && data[4] == 0x27 && data[5] == 0x1c {
|
||
return "7z"
|
||
}
|
||
// RAR 5.0 signature: 52 61 72 21 1A 07 01 00 ("Rar!\x1a\x07\x01\x00")
|
||
// RAR 1.5–4.x: 52 61 72 21 1A 07 00 ("Rar!\x1a\x07\x00")
|
||
if len(data) >= 7 && data[0] == 'R' && data[1] == 'a' && data[2] == 'r' && data[3] == '!' && data[4] == 0x1a && data[5] == 0x07 {
|
||
return "rar"
|
||
}
|
||
// xz signature: FD 37 7A 58 5A 00
|
||
if len(data) >= 6 && data[0] == 0xfd && data[1] == '7' && data[2] == 'z' && data[3] == 'X' && data[4] == 'Z' && data[5] == 0x00 {
|
||
return "tar.xz"
|
||
}
|
||
if strings.HasSuffix(lower, ".tar.gz") || strings.HasSuffix(lower, ".tgz") {
|
||
return "tar.gz"
|
||
}
|
||
if strings.HasSuffix(lower, ".tar.bz2") || strings.HasSuffix(lower, ".tbz2") || strings.HasSuffix(lower, ".tbz") {
|
||
return "tar.bz2"
|
||
}
|
||
if strings.HasSuffix(lower, ".tar.xz") || strings.HasSuffix(lower, ".txz") {
|
||
return "tar.xz"
|
||
}
|
||
if strings.HasSuffix(lower, ".tar") {
|
||
return "tar"
|
||
}
|
||
if strings.HasSuffix(lower, ".zip") {
|
||
return "zip"
|
||
}
|
||
if strings.HasSuffix(lower, ".7z") {
|
||
return "7z"
|
||
}
|
||
if strings.HasSuffix(lower, ".rar") {
|
||
return "rar"
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func walkZip(data []byte, fn EntryFunc) (int64, int64, error) {
|
||
zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
|
||
if err != nil {
|
||
return 0, 0, fmt.Errorf("zip open: %w", err)
|
||
}
|
||
var entries, bytesOut int64
|
||
var totalBytes int64
|
||
for _, f := range zr.File {
|
||
if entries >= MaxEntries {
|
||
return entries, bytesOut, fmt.Errorf("archive has more than %d entries", int64(MaxEntries))
|
||
}
|
||
isDir := f.FileInfo().IsDir() || strings.HasSuffix(f.Name, "/")
|
||
if isDir {
|
||
if err := fn(f.Name, int64(f.Mode()), true, nil); err != nil {
|
||
return entries, bytesOut, err
|
||
}
|
||
entries++
|
||
continue
|
||
}
|
||
rc, err := f.Open()
|
||
if err != nil {
|
||
return entries, bytesOut, fmt.Errorf("zip entry %q: %w", f.Name, err)
|
||
}
|
||
// Cap each entry by `EntrySoftBytes` so a malformed/zip-bomb
|
||
// entry can't run away with memory. We DO NOT trust
|
||
// f.UncompressedSize64 — some zip writers leave it 0 for streamed
|
||
// entries; using it for accounting (bytesOut += size) caused the
|
||
// per-archive total to be charged the 4 GB soft cap per such
|
||
// entry, tripping MaxTotalBytes after just a couple of
|
||
// files and aborting the extract mid-stream — silently leaving
|
||
// the first N files extracted and the rest dropped (this was the
|
||
// root cause of the 7DTD mod-upload "Config/ folders missing"
|
||
// bug). Use a counting reader so accounting reflects real bytes.
|
||
cr := &countingReader{r: io.LimitReader(rc, EntrySoftBytes)}
|
||
if err := fn(f.Name, int64(f.Mode()), false, cr); err != nil {
|
||
_ = rc.Close()
|
||
return entries, bytesOut, err
|
||
}
|
||
_ = rc.Close()
|
||
entries++
|
||
bytesOut += cr.n
|
||
totalBytes += cr.n
|
||
if totalBytes > MaxTotalBytes {
|
||
return entries, bytesOut, fmt.Errorf("archive uncompressed size exceeds %d bytes", int64(MaxTotalBytes))
|
||
}
|
||
}
|
||
return entries, bytesOut, nil
|
||
}
|
||
|
||
// countingReader wraps an io.Reader and tracks the byte count actually
|
||
// consumed. Used by walkZip / walkTar accounting so the per-archive
|
||
// total reflects real-world bytes, not zip-header declared sizes (which
|
||
// some encoders leave as 0).
|
||
type countingReader struct {
|
||
r io.Reader
|
||
n int64
|
||
}
|
||
|
||
func (c *countingReader) Read(p []byte) (int, error) {
|
||
n, err := c.r.Read(p)
|
||
c.n += int64(n)
|
||
return n, err
|
||
}
|
||
|
||
func walkTar(r io.Reader, fn EntryFunc) (int64, int64, error) {
|
||
tr := tar.NewReader(r)
|
||
var entries, bytesOut, totalBytes int64
|
||
for {
|
||
if entries >= MaxEntries {
|
||
return entries, bytesOut, fmt.Errorf("archive has more than %d entries", int64(MaxEntries))
|
||
}
|
||
hdr, err := tr.Next()
|
||
if err == io.EOF {
|
||
return entries, bytesOut, nil
|
||
}
|
||
if err != nil {
|
||
return entries, bytesOut, fmt.Errorf("tar next: %w", err)
|
||
}
|
||
switch hdr.Typeflag {
|
||
case tar.TypeDir:
|
||
if err := fn(hdr.Name, hdr.Mode, true, nil); err != nil {
|
||
return entries, bytesOut, err
|
||
}
|
||
entries++
|
||
case tar.TypeReg, tar.TypeRegA:
|
||
if err := fn(hdr.Name, hdr.Mode, false, tr); err != nil {
|
||
return entries, bytesOut, err
|
||
}
|
||
entries++
|
||
bytesOut += hdr.Size
|
||
totalBytes += hdr.Size
|
||
if totalBytes > MaxTotalBytes {
|
||
return entries, bytesOut, fmt.Errorf("archive uncompressed size exceeds %d bytes", int64(MaxTotalBytes))
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func walk7z(data []byte, fn EntryFunc) (int64, int64, error) {
|
||
zr, err := sevenzip.NewReader(bytes.NewReader(data), int64(len(data)))
|
||
if err != nil {
|
||
return 0, 0, fmt.Errorf("7z open: %w", err)
|
||
}
|
||
var entries, bytesOut, totalBytes int64
|
||
for _, f := range zr.File {
|
||
if entries >= MaxEntries {
|
||
return entries, bytesOut, fmt.Errorf("archive has more than %d entries", int64(MaxEntries))
|
||
}
|
||
isDir := f.FileInfo().IsDir() || strings.HasSuffix(f.Name, "/")
|
||
if isDir {
|
||
if err := fn(f.Name, int64(f.Mode()), true, nil); err != nil {
|
||
return entries, bytesOut, err
|
||
}
|
||
entries++
|
||
continue
|
||
}
|
||
rc, err := f.Open()
|
||
if err != nil {
|
||
return entries, bytesOut, fmt.Errorf("7z entry %q: %w", f.Name, err)
|
||
}
|
||
// See countingReader rationale in walkZip — header-declared
|
||
// size can be 0 for streamed entries, charging the soft-cap per
|
||
// entry would falsely abort the extract.
|
||
cr := &countingReader{r: io.LimitReader(rc, EntrySoftBytes)}
|
||
if err := fn(f.Name, int64(f.Mode()), false, cr); err != nil {
|
||
_ = rc.Close()
|
||
return entries, bytesOut, err
|
||
}
|
||
_ = rc.Close()
|
||
entries++
|
||
bytesOut += cr.n
|
||
totalBytes += cr.n
|
||
if totalBytes > MaxTotalBytes {
|
||
return entries, bytesOut, fmt.Errorf("archive uncompressed size exceeds %d bytes", int64(MaxTotalBytes))
|
||
}
|
||
}
|
||
return entries, bytesOut, nil
|
||
}
|
||
|
||
func walkRar(data []byte, fn EntryFunc) (int64, int64, error) {
|
||
rr, err := rardecode.NewReader(bytes.NewReader(data))
|
||
if err != nil {
|
||
return 0, 0, fmt.Errorf("rar open: %w", err)
|
||
}
|
||
var entries, bytesOut, totalBytes int64
|
||
for {
|
||
if entries >= MaxEntries {
|
||
return entries, bytesOut, fmt.Errorf("archive has more than %d entries", int64(MaxEntries))
|
||
}
|
||
hdr, err := rr.Next()
|
||
if errors.Is(err, io.EOF) {
|
||
return entries, bytesOut, nil
|
||
}
|
||
if err != nil {
|
||
return entries, bytesOut, fmt.Errorf("rar next: %w", err)
|
||
}
|
||
mode := int64(hdr.Mode().Perm())
|
||
if hdr.IsDir {
|
||
if err := fn(hdr.Name, mode, true, nil); err != nil {
|
||
return entries, bytesOut, err
|
||
}
|
||
entries++
|
||
continue
|
||
}
|
||
// See countingReader rationale in walkZip.
|
||
cr := &countingReader{r: io.LimitReader(rr, EntrySoftBytes)}
|
||
if err := fn(hdr.Name, mode, false, cr); err != nil {
|
||
return entries, bytesOut, err
|
||
}
|
||
entries++
|
||
bytesOut += cr.n
|
||
totalBytes += cr.n
|
||
if totalBytes > MaxTotalBytes {
|
||
return entries, bytesOut, fmt.Errorf("archive uncompressed size exceeds %d bytes", int64(MaxTotalBytes))
|
||
}
|
||
}
|
||
}
|
||
|
||
// SafeEntryPath strips leading slashes / drive letters and
|
||
// rejects any path that tries to ../ out of the destination. Returns
|
||
// "" on rejected entries — caller skips them silently.
|
||
func SafeEntryPath(name string) string {
|
||
if name == "" {
|
||
return ""
|
||
}
|
||
// Normalize separators + strip leading drive / slash.
|
||
clean := strings.ReplaceAll(name, "\\", "/")
|
||
for strings.HasPrefix(clean, "/") {
|
||
clean = clean[1:]
|
||
}
|
||
if len(clean) >= 2 && clean[1] == ':' {
|
||
clean = clean[2:]
|
||
}
|
||
clean = path.Clean(clean)
|
||
if clean == "." || clean == "" {
|
||
return ""
|
||
}
|
||
if strings.HasPrefix(clean, "../") || clean == ".." || strings.Contains(clean, "/../") {
|
||
return ""
|
||
}
|
||
return clean
|
||
}
|