blob: f91a5f2cbef67bd708d9ba972bf8dde899107ea2 [file] [log] [blame]
Colin Cross2fe66872015-03-30 17:20:39 -07001// Copyright 2015 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
Jeff Gaston11b5c512017-10-12 12:19:14 -070015package zip
Colin Cross2fe66872015-03-30 17:20:39 -070016
17import (
Dan Willemsen017d8932016-08-04 15:43:03 -070018 "bytes"
19 "compress/flate"
Zhenhuang Wangb8451b82023-01-06 20:58:01 +080020 "crypto/sha256"
21 "encoding/binary"
Jeff Gastona2976952017-08-22 17:51:25 -070022 "errors"
Colin Cross2fe66872015-03-30 17:20:39 -070023 "fmt"
Zhenhuang Wangb8451b82023-01-06 20:58:01 +080024 "hash"
Dan Willemsen017d8932016-08-04 15:43:03 -070025 "hash/crc32"
Colin Cross2fe66872015-03-30 17:20:39 -070026 "io"
27 "io/ioutil"
28 "os"
29 "path/filepath"
Jeff Gastona2976952017-08-22 17:51:25 -070030 "sort"
Colin Cross2fe66872015-03-30 17:20:39 -070031 "strings"
Dan Willemsen017d8932016-08-04 15:43:03 -070032 "sync"
Colin Cross1d98ee22018-09-18 17:05:15 -070033 "syscall"
Colin Cross2fe66872015-03-30 17:20:39 -070034 "time"
Colin Crossfd708b52021-03-23 14:16:05 -070035
36 "android/soong/response"
Dan Willemsen017d8932016-08-04 15:43:03 -070037
Colin Crossf83c1502017-11-10 13:11:02 -080038 "github.com/google/blueprint/pathtools"
39
Jeff Gastona2976952017-08-22 17:51:25 -070040 "android/soong/jar"
Dan Willemsen017d8932016-08-04 15:43:03 -070041 "android/soong/third_party/zip"
Colin Cross2fe66872015-03-30 17:20:39 -070042)
43
Zhenhuang Wangb8451b82023-01-06 20:58:01 +080044// Sha256HeaderID is a custom Header ID for the `extra` field in
45// the file header to store the SHA checksum.
46const Sha256HeaderID = 0x4967
47
48// Sha256HeaderSignature is the signature to verify that the extra
49// data block is used to store the SHA checksum.
50const Sha256HeaderSignature = 0x9514
51
Dan Willemsen017d8932016-08-04 15:43:03 -070052// Block size used during parallel compression of a single file.
53const parallelBlockSize = 1 * 1024 * 1024 // 1MB
54
55// Minimum file size to use parallel compression. It requires more
56// flate.Writer allocations, since we can't change the dictionary
57// during Reset
58const minParallelFileSize = parallelBlockSize * 6
59
60// Size of the ZIP compression window (32KB)
61const windowSize = 32 * 1024
62
63type nopCloser struct {
64 io.Writer
65}
66
67func (nopCloser) Close() error {
68 return nil
69}
70
Jeff Gastoncef50b92017-08-23 15:41:35 -070071type byteReaderCloser struct {
Colin Cross635acc92017-09-12 22:50:46 -070072 *bytes.Reader
Jeff Gastoncef50b92017-08-23 15:41:35 -070073 io.Closer
74}
75
Nan Zhang9067b042017-03-17 14:04:43 -070076type pathMapping struct {
77 dest, src string
Nan Zhangf281bd82017-04-25 16:47:45 -070078 zipMethod uint16
79}
80
Jeff Gastonc3bdc972017-10-12 12:18:19 -070081type FileArg struct {
82 PathPrefixInZip, SourcePrefixToStrip string
Colin Cross25ff3052023-05-08 15:05:29 -070083 ExplicitPathInZip string
Jeff Gastonc3bdc972017-10-12 12:18:19 -070084 SourceFiles []string
Colin Crossb7c69112018-09-18 16:51:43 -070085 JunkPaths bool
Jeff Gastonc3bdc972017-10-12 12:18:19 -070086 GlobDir string
87}
88
Colin Crossfe945b42018-09-27 15:00:07 -070089type FileArgsBuilder struct {
90 state FileArg
91 err error
92 fs pathtools.FileSystem
93
94 fileArgs []FileArg
95}
96
97func NewFileArgsBuilder() *FileArgsBuilder {
98 return &FileArgsBuilder{
99 fs: pathtools.OsFs,
100 }
101}
102
103func (b *FileArgsBuilder) JunkPaths(v bool) *FileArgsBuilder {
104 b.state.JunkPaths = v
105 b.state.SourcePrefixToStrip = ""
106 return b
107}
108
109func (b *FileArgsBuilder) SourcePrefixToStrip(prefixToStrip string) *FileArgsBuilder {
110 b.state.JunkPaths = false
111 b.state.SourcePrefixToStrip = prefixToStrip
112 return b
113}
114
115func (b *FileArgsBuilder) PathPrefixInZip(rootPrefix string) *FileArgsBuilder {
116 b.state.PathPrefixInZip = rootPrefix
117 return b
118}
119
120func (b *FileArgsBuilder) File(name string) *FileArgsBuilder {
121 if b.err != nil {
122 return b
123 }
124
125 arg := b.state
126 arg.SourceFiles = []string{name}
127 b.fileArgs = append(b.fileArgs, arg)
Colin Cross25ff3052023-05-08 15:05:29 -0700128
129 if b.state.ExplicitPathInZip != "" {
130 b.state.ExplicitPathInZip = ""
131 }
Colin Crossfe945b42018-09-27 15:00:07 -0700132 return b
133}
134
135func (b *FileArgsBuilder) Dir(name string) *FileArgsBuilder {
136 if b.err != nil {
137 return b
138 }
139
140 arg := b.state
141 arg.GlobDir = name
142 b.fileArgs = append(b.fileArgs, arg)
143 return b
144}
145
Sasha Smundak8eedba62020-11-16 19:00:27 -0800146// List reads the file names from the given file and adds them to the source files list.
Colin Crossfe945b42018-09-27 15:00:07 -0700147func (b *FileArgsBuilder) List(name string) *FileArgsBuilder {
148 if b.err != nil {
149 return b
150 }
151
152 f, err := b.fs.Open(name)
153 if err != nil {
154 b.err = err
155 return b
156 }
157 defer f.Close()
158
159 list, err := ioutil.ReadAll(f)
160 if err != nil {
161 b.err = err
162 return b
163 }
164
165 arg := b.state
Jiyong Park04bbf982019-11-04 13:18:41 +0900166 arg.SourceFiles = strings.Fields(string(list))
Colin Crossfe945b42018-09-27 15:00:07 -0700167 b.fileArgs = append(b.fileArgs, arg)
168 return b
169}
170
Sasha Smundak8eedba62020-11-16 19:00:27 -0800171// RspFile reads the file names from given .rsp file and adds them to the source files list.
Colin Cross053fca12020-08-19 13:51:47 -0700172func (b *FileArgsBuilder) RspFile(name string) *FileArgsBuilder {
173 if b.err != nil {
174 return b
175 }
176
177 f, err := b.fs.Open(name)
178 if err != nil {
179 b.err = err
180 return b
181 }
182 defer f.Close()
183
Colin Crossfd708b52021-03-23 14:16:05 -0700184 arg := b.state
185 arg.SourceFiles, err = response.ReadRspFile(f)
Colin Cross053fca12020-08-19 13:51:47 -0700186 if err != nil {
187 b.err = err
188 return b
189 }
Colin Crosscaf4d4c2021-02-03 15:15:14 -0800190 for i := range arg.SourceFiles {
191 arg.SourceFiles[i] = pathtools.MatchEscape(arg.SourceFiles[i])
192 }
Colin Cross053fca12020-08-19 13:51:47 -0700193 b.fileArgs = append(b.fileArgs, arg)
194 return b
195}
196
Colin Cross25ff3052023-05-08 15:05:29 -0700197// ExplicitPathInZip sets the path in the zip file for the next File call.
198func (b *FileArgsBuilder) ExplicitPathInZip(s string) *FileArgsBuilder {
199 b.state.ExplicitPathInZip = s
200 return b
201}
202
Colin Crossfe945b42018-09-27 15:00:07 -0700203func (b *FileArgsBuilder) Error() error {
204 if b == nil {
205 return nil
206 }
207 return b.err
208}
209
210func (b *FileArgsBuilder) FileArgs() []FileArg {
211 if b == nil {
212 return nil
213 }
214 return b.fileArgs
215}
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700216
Colin Cross1d98ee22018-09-18 17:05:15 -0700217type IncorrectRelativeRootError struct {
218 RelativeRoot string
219 Path string
220}
221
222func (x IncorrectRelativeRootError) Error() string {
223 return fmt.Sprintf("path %q is outside relative root %q", x.Path, x.RelativeRoot)
224}
225
Colin Cross7ddd08a2022-08-15 15:47:41 -0700226type ConflictingFileError struct {
227 Dest string
228 Prev string
229 Src string
230}
231
232func (x ConflictingFileError) Error() string {
233 return fmt.Sprintf("destination %q has two files %q and %q", x.Dest, x.Prev, x.Src)
234}
235
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700236type ZipWriter struct {
Colin Crosse5580972017-08-30 17:40:21 -0700237 time time.Time
238 createdFiles map[string]string
239 createdDirs map[string]string
240 directories bool
Colin Crosse19c7932015-04-24 15:08:38 -0700241
Dan Willemsen017d8932016-08-04 15:43:03 -0700242 errors chan error
243 writeOps chan chan *zipEntry
244
Jeff Gaston175f34c2017-08-17 21:43:21 -0700245 cpuRateLimiter *CPURateLimiter
246 memoryRateLimiter *MemoryRateLimiter
Dan Willemsen017d8932016-08-04 15:43:03 -0700247
248 compressorPool sync.Pool
249 compLevel int
Colin Cross05518bc2018-09-27 15:06:19 -0700250
Colin Cross4be8f9e2018-09-28 15:16:48 -0700251 followSymlinks pathtools.ShouldFollowSymlinks
252 ignoreMissingFiles bool
Colin Cross09f11052018-09-21 15:12:39 -0700253
Colin Cross4be8f9e2018-09-28 15:16:48 -0700254 stderr io.Writer
255 fs pathtools.FileSystem
Zhenhuang Wangb8451b82023-01-06 20:58:01 +0800256
257 sha256Checksum bool
Dan Willemsen017d8932016-08-04 15:43:03 -0700258}
259
260type zipEntry struct {
261 fh *zip.FileHeader
262
263 // List of delayed io.Reader
264 futureReaders chan chan io.Reader
Jeff Gaston175f34c2017-08-17 21:43:21 -0700265
266 // Only used for passing into the MemoryRateLimiter to ensure we
267 // release as much memory as much as we request
268 allocatedSize int64
Colin Cross2fe66872015-03-30 17:20:39 -0700269}
270
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700271type ZipArgs struct {
Colin Crossfe945b42018-09-27 15:00:07 -0700272 FileArgs []FileArg
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700273 OutputFilePath string
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700274 EmulateJar bool
Colin Cross9cb51db2019-06-17 14:12:41 -0700275 SrcJar bool
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700276 AddDirectoryEntriesToZip bool
277 CompressionLevel int
278 ManifestSourcePath string
279 NumParallelJobs int
280 NonDeflatedFiles map[string]bool
Colin Crossf83c1502017-11-10 13:11:02 -0800281 WriteIfChanged bool
Colin Cross09f11052018-09-21 15:12:39 -0700282 StoreSymlinks bool
Colin Cross4be8f9e2018-09-28 15:16:48 -0700283 IgnoreMissingFiles bool
Zhenhuang Wangb8451b82023-01-06 20:58:01 +0800284 Sha256Checksum bool
Dan Albert1cec6d52023-07-13 17:41:04 +0000285 DoNotWrite bool
Jihoon Kang3730d7e2023-09-12 23:52:36 +0000286 Quiet bool
Colin Cross09f11052018-09-21 15:12:39 -0700287
Colin Cross4be8f9e2018-09-28 15:16:48 -0700288 Stderr io.Writer
Colin Cross09f11052018-09-21 15:12:39 -0700289 Filesystem pathtools.FileSystem
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700290}
Colin Cross2fe66872015-03-30 17:20:39 -0700291
Sasha Smundak8eedba62020-11-16 19:00:27 -0800292func zipTo(args ZipArgs, w io.Writer) error {
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700293 if args.EmulateJar {
294 args.AddDirectoryEntriesToZip = true
Jeff Gaston8edbb3a2017-08-22 20:05:28 -0700295 }
296
Colin Cross09f11052018-09-21 15:12:39 -0700297 // Have Glob follow symlinks if they are not being stored as symlinks in the zip file.
298 followSymlinks := pathtools.ShouldFollowSymlinks(!args.StoreSymlinks)
299
Colin Cross05518bc2018-09-27 15:06:19 -0700300 z := &ZipWriter{
Colin Cross4be8f9e2018-09-28 15:16:48 -0700301 time: jar.DefaultTime,
302 createdDirs: make(map[string]string),
303 createdFiles: make(map[string]string),
304 directories: args.AddDirectoryEntriesToZip,
305 compLevel: args.CompressionLevel,
306 followSymlinks: followSymlinks,
307 ignoreMissingFiles: args.IgnoreMissingFiles,
308 stderr: args.Stderr,
309 fs: args.Filesystem,
Zhenhuang Wangb8451b82023-01-06 20:58:01 +0800310 sha256Checksum: args.Sha256Checksum,
Colin Cross2fe66872015-03-30 17:20:39 -0700311 }
Colin Cross05518bc2018-09-27 15:06:19 -0700312
313 if z.fs == nil {
314 z.fs = pathtools.OsFs
315 }
316
Colin Cross4be8f9e2018-09-28 15:16:48 -0700317 if z.stderr == nil {
318 z.stderr = os.Stderr
319 }
320
Nan Zhang9067b042017-03-17 14:04:43 -0700321 pathMappings := []pathMapping{}
Nan Zhang9067b042017-03-17 14:04:43 -0700322
Colin Crossd3216292018-09-14 15:06:31 -0700323 noCompression := args.CompressionLevel == 0
324
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700325 for _, fa := range args.FileArgs {
Colin Cross1d98ee22018-09-18 17:05:15 -0700326 var srcs []string
327 for _, s := range fa.SourceFiles {
328 s = strings.TrimSpace(s)
329 if s == "" {
330 continue
331 }
332
Colin Cross82ea3fb2021-04-05 17:48:26 -0700333 result, err := z.fs.Glob(s, nil, followSymlinks)
Colin Cross1d98ee22018-09-18 17:05:15 -0700334 if err != nil {
335 return err
336 }
Colin Cross82ea3fb2021-04-05 17:48:26 -0700337 if len(result.Matches) == 0 {
Colin Cross4be8f9e2018-09-28 15:16:48 -0700338 err := &os.PathError{
339 Op: "lstat",
Colin Cross1d98ee22018-09-18 17:05:15 -0700340 Path: s,
341 Err: os.ErrNotExist,
342 }
Colin Cross4be8f9e2018-09-28 15:16:48 -0700343 if args.IgnoreMissingFiles {
Jihoon Kang3730d7e2023-09-12 23:52:36 +0000344 if !args.Quiet {
345 fmt.Fprintln(z.stderr, "warning:", err)
346 }
Colin Cross4be8f9e2018-09-28 15:16:48 -0700347 } else {
348 return err
349 }
Colin Cross1d98ee22018-09-18 17:05:15 -0700350 }
Colin Cross82ea3fb2021-04-05 17:48:26 -0700351 srcs = append(srcs, result.Matches...)
Colin Cross1d98ee22018-09-18 17:05:15 -0700352 }
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700353 if fa.GlobDir != "" {
Colin Cross1d98ee22018-09-18 17:05:15 -0700354 if exists, isDir, err := z.fs.Exists(fa.GlobDir); err != nil {
355 return err
Colin Cross4be8f9e2018-09-28 15:16:48 -0700356 } else if !exists && !args.IgnoreMissingFiles {
357 err := &os.PathError{
358 Op: "lstat",
Colin Cross1d98ee22018-09-18 17:05:15 -0700359 Path: fa.GlobDir,
360 Err: os.ErrNotExist,
361 }
Colin Cross4be8f9e2018-09-28 15:16:48 -0700362 if args.IgnoreMissingFiles {
Jihoon Kang3730d7e2023-09-12 23:52:36 +0000363 if !args.Quiet {
364 fmt.Fprintln(z.stderr, "warning:", err)
365 }
Colin Cross4be8f9e2018-09-28 15:16:48 -0700366 } else {
367 return err
368 }
369 } else if !isDir && !args.IgnoreMissingFiles {
370 err := &os.PathError{
371 Op: "lstat",
Colin Cross1d98ee22018-09-18 17:05:15 -0700372 Path: fa.GlobDir,
373 Err: syscall.ENOTDIR,
374 }
Colin Cross4be8f9e2018-09-28 15:16:48 -0700375 if args.IgnoreMissingFiles {
Jihoon Kang3730d7e2023-09-12 23:52:36 +0000376 if !args.Quiet {
377 fmt.Fprintln(z.stderr, "warning:", err)
378 }
Colin Cross4be8f9e2018-09-28 15:16:48 -0700379 } else {
380 return err
381 }
Colin Cross1d98ee22018-09-18 17:05:15 -0700382 }
Colin Cross82ea3fb2021-04-05 17:48:26 -0700383 result, err := z.fs.Glob(filepath.Join(fa.GlobDir, "**/*"), nil, followSymlinks)
Colin Cross1d98ee22018-09-18 17:05:15 -0700384 if err != nil {
385 return err
386 }
Colin Cross82ea3fb2021-04-05 17:48:26 -0700387 srcs = append(srcs, result.Matches...)
Colin Cross7b10cf12017-08-30 14:12:21 -0700388 }
389 for _, src := range srcs {
Colin Crossb7c69112018-09-18 16:51:43 -0700390 err := fillPathPairs(fa, src, &pathMappings, args.NonDeflatedFiles, noCompression)
Colin Crossd3216292018-09-14 15:06:31 -0700391 if err != nil {
Colin Cross05518bc2018-09-27 15:06:19 -0700392 return err
Nan Zhang9067b042017-03-17 14:04:43 -0700393 }
394 }
395 }
396
Colin Cross9cb51db2019-06-17 14:12:41 -0700397 return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.SrcJar, args.NumParallelJobs)
Colin Cross05518bc2018-09-27 15:06:19 -0700398}
399
Sasha Smundak8eedba62020-11-16 19:00:27 -0800400// Zip creates an output zip archive from given sources.
Colin Cross05518bc2018-09-27 15:06:19 -0700401func Zip(args ZipArgs) error {
402 if args.OutputFilePath == "" {
403 return fmt.Errorf("output file path must be nonempty")
404 }
405
Colin Crossf83c1502017-11-10 13:11:02 -0800406 buf := &bytes.Buffer{}
407 var out io.Writer = buf
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700408
Colin Cross665b6782021-03-15 15:28:40 -0700409 var zipErr error
410
Dan Albert1cec6d52023-07-13 17:41:04 +0000411 if args.DoNotWrite {
412 out = io.Discard
413 } else if !args.WriteIfChanged {
Colin Crossf83c1502017-11-10 13:11:02 -0800414 f, err := os.Create(args.OutputFilePath)
415 if err != nil {
416 return err
417 }
418
419 defer f.Close()
420 defer func() {
Colin Cross665b6782021-03-15 15:28:40 -0700421 if zipErr != nil {
Colin Crossf83c1502017-11-10 13:11:02 -0800422 os.Remove(args.OutputFilePath)
423 }
424 }()
425
426 out = f
427 }
428
Colin Cross665b6782021-03-15 15:28:40 -0700429 zipErr = zipTo(args, out)
430 if zipErr != nil {
431 return zipErr
Colin Crossf83c1502017-11-10 13:11:02 -0800432 }
433
Dan Albert1cec6d52023-07-13 17:41:04 +0000434 if args.WriteIfChanged && !args.DoNotWrite {
Colin Crossf83c1502017-11-10 13:11:02 -0800435 err := pathtools.WriteFileIfChanged(args.OutputFilePath, buf.Bytes(), 0666)
436 if err != nil {
437 return err
438 }
439 }
440
441 return nil
Colin Cross2fe66872015-03-30 17:20:39 -0700442}
443
Colin Crossb7c69112018-09-18 16:51:43 -0700444func fillPathPairs(fa FileArg, src string, pathMappings *[]pathMapping,
Colin Crossd3216292018-09-14 15:06:31 -0700445 nonDeflatedFiles map[string]bool, noCompression bool) error {
446
Colin Crossb7c69112018-09-18 16:51:43 -0700447 var dest string
448
Colin Cross25ff3052023-05-08 15:05:29 -0700449 if fa.ExplicitPathInZip != "" {
450 dest = fa.ExplicitPathInZip
451 } else if fa.JunkPaths {
Colin Crossb7c69112018-09-18 16:51:43 -0700452 dest = filepath.Base(src)
453 } else {
454 var err error
455 dest, err = filepath.Rel(fa.SourcePrefixToStrip, src)
456 if err != nil {
457 return err
458 }
Colin Cross1d98ee22018-09-18 17:05:15 -0700459 if strings.HasPrefix(dest, "../") {
460 return IncorrectRelativeRootError{
461 Path: src,
462 RelativeRoot: fa.SourcePrefixToStrip,
463 }
464 }
Nan Zhang9067b042017-03-17 14:04:43 -0700465 }
Colin Crossb7c69112018-09-18 16:51:43 -0700466 dest = filepath.Join(fa.PathPrefixInZip, dest)
Nan Zhang9067b042017-03-17 14:04:43 -0700467
Nan Zhangf281bd82017-04-25 16:47:45 -0700468 zipMethod := zip.Deflate
Colin Crossd3216292018-09-14 15:06:31 -0700469 if _, found := nonDeflatedFiles[dest]; found || noCompression {
Nan Zhangf281bd82017-04-25 16:47:45 -0700470 zipMethod = zip.Store
471 }
472 *pathMappings = append(*pathMappings,
473 pathMapping{dest: dest, src: src, zipMethod: zipMethod})
Nan Zhang9067b042017-03-17 14:04:43 -0700474
475 return nil
476}
477
Jeff Gastona2976952017-08-22 17:51:25 -0700478func jarSort(mappings []pathMapping) {
Sasha Smundak8eedba62020-11-16 19:00:27 -0800479 sort.SliceStable(mappings, func(i int, j int) bool {
Jeff Gastona2976952017-08-22 17:51:25 -0700480 return jar.EntryNamesLess(mappings[i].dest, mappings[j].dest)
Sasha Smundak8eedba62020-11-16 19:00:27 -0800481 })
Jeff Gastona2976952017-08-22 17:51:25 -0700482}
483
Colin Cross9cb51db2019-06-17 14:12:41 -0700484func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar, srcJar bool,
485 parallelJobs int) error {
486
Dan Willemsen017d8932016-08-04 15:43:03 -0700487 z.errors = make(chan error)
488 defer close(z.errors)
Colin Cross2fe66872015-03-30 17:20:39 -0700489
Dan Willemsen017d8932016-08-04 15:43:03 -0700490 // This channel size can be essentially unlimited -- it's used as a fifo
491 // queue decouple the CPU and IO loads. Directories don't require any
492 // compression time, but still cost some IO. Similar with small files that
493 // can be very fast to compress. Some files that are more difficult to
494 // compress won't take a corresponding longer time writing out.
495 //
496 // The optimum size here depends on your CPU and IO characteristics, and
497 // the the layout of your zip file. 1000 was chosen mostly at random as
498 // something that worked reasonably well for a test file.
499 //
500 // The RateLimit object will put the upper bounds on the number of
501 // parallel compressions and outstanding buffers.
502 z.writeOps = make(chan chan *zipEntry, 1000)
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700503 z.cpuRateLimiter = NewCPURateLimiter(int64(parallelJobs))
Jeff Gaston175f34c2017-08-17 21:43:21 -0700504 z.memoryRateLimiter = NewMemoryRateLimiter(0)
505 defer func() {
506 z.cpuRateLimiter.Stop()
507 z.memoryRateLimiter.Stop()
508 }()
Jeff Gastona2976952017-08-22 17:51:25 -0700509
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700510 if manifest != "" && !emulateJar {
Colin Cross635acc92017-09-12 22:50:46 -0700511 return errors.New("must specify --jar when specifying a manifest via -m")
Jeff Gastona2976952017-08-22 17:51:25 -0700512 }
513
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700514 if emulateJar {
Colin Cross635acc92017-09-12 22:50:46 -0700515 // manifest may be empty, in which case addManifest will fill in a default
516 pathMappings = append(pathMappings, pathMapping{jar.ManifestFile, manifest, zip.Deflate})
517
Jeff Gastona2976952017-08-22 17:51:25 -0700518 jarSort(pathMappings)
519 }
520
Dan Willemsen017d8932016-08-04 15:43:03 -0700521 go func() {
522 var err error
523 defer close(z.writeOps)
524
Nan Zhang9067b042017-03-17 14:04:43 -0700525 for _, ele := range pathMappings {
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700526 if emulateJar && ele.dest == jar.ManifestFile {
Jeff Gastoncef50b92017-08-23 15:41:35 -0700527 err = z.addManifest(ele.dest, ele.src, ele.zipMethod)
528 } else {
Colin Cross9cb51db2019-06-17 14:12:41 -0700529 err = z.addFile(ele.dest, ele.src, ele.zipMethod, emulateJar, srcJar)
Jeff Gastoncef50b92017-08-23 15:41:35 -0700530 }
Dan Willemsen017d8932016-08-04 15:43:03 -0700531 if err != nil {
532 z.errors <- err
533 return
534 }
535 }
Dan Willemsen017d8932016-08-04 15:43:03 -0700536 }()
537
538 zipw := zip.NewWriter(f)
539
540 var currentWriteOpChan chan *zipEntry
541 var currentWriter io.WriteCloser
542 var currentReaders chan chan io.Reader
543 var currentReader chan io.Reader
544 var done bool
545
546 for !done {
547 var writeOpsChan chan chan *zipEntry
548 var writeOpChan chan *zipEntry
549 var readersChan chan chan io.Reader
550
551 if currentReader != nil {
552 // Only read and process errors
553 } else if currentReaders != nil {
554 readersChan = currentReaders
555 } else if currentWriteOpChan != nil {
556 writeOpChan = currentWriteOpChan
557 } else {
558 writeOpsChan = z.writeOps
559 }
560
561 select {
562 case writeOp, ok := <-writeOpsChan:
563 if !ok {
564 done = true
565 }
566
567 currentWriteOpChan = writeOp
568
569 case op := <-writeOpChan:
570 currentWriteOpChan = nil
571
Colin Crossf83c1502017-11-10 13:11:02 -0800572 var err error
Dan Willemsen017d8932016-08-04 15:43:03 -0700573 if op.fh.Method == zip.Deflate {
574 currentWriter, err = zipw.CreateCompressedHeader(op.fh)
575 } else {
576 var zw io.Writer
Jeff Gastonc5eb66d2017-08-24 14:11:27 -0700577
578 op.fh.CompressedSize64 = op.fh.UncompressedSize64
579
580 zw, err = zipw.CreateHeaderAndroid(op.fh)
Dan Willemsen017d8932016-08-04 15:43:03 -0700581 currentWriter = nopCloser{zw}
582 }
583 if err != nil {
584 return err
585 }
586
587 currentReaders = op.futureReaders
588 if op.futureReaders == nil {
589 currentWriter.Close()
590 currentWriter = nil
591 }
Jeff Gaston175f34c2017-08-17 21:43:21 -0700592 z.memoryRateLimiter.Finish(op.allocatedSize)
Dan Willemsen017d8932016-08-04 15:43:03 -0700593
594 case futureReader, ok := <-readersChan:
595 if !ok {
596 // Done with reading
597 currentWriter.Close()
598 currentWriter = nil
599 currentReaders = nil
600 }
601
602 currentReader = futureReader
603
604 case reader := <-currentReader:
Colin Crossf83c1502017-11-10 13:11:02 -0800605 _, err := io.Copy(currentWriter, reader)
Dan Willemsen017d8932016-08-04 15:43:03 -0700606 if err != nil {
607 return err
608 }
Dan Willemsen017d8932016-08-04 15:43:03 -0700609
610 currentReader = nil
611
Colin Crossf83c1502017-11-10 13:11:02 -0800612 case err := <-z.errors:
Colin Cross2fe66872015-03-30 17:20:39 -0700613 return err
614 }
615 }
616
Dan Willemsen017d8932016-08-04 15:43:03 -0700617 // One last chance to catch an error
618 select {
Colin Crossf83c1502017-11-10 13:11:02 -0800619 case err := <-z.errors:
Dan Willemsen017d8932016-08-04 15:43:03 -0700620 return err
621 default:
622 zipw.Close()
623 return nil
Colin Cross2fe66872015-03-30 17:20:39 -0700624 }
Colin Cross2fe66872015-03-30 17:20:39 -0700625}
626
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700627// imports (possibly with compression) <src> into the zip at sub-path <dest>
Colin Cross9cb51db2019-06-17 14:12:41 -0700628func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar, srcJar bool) error {
Dan Willemsen017d8932016-08-04 15:43:03 -0700629 var fileSize int64
Dan Willemsen10462b32017-03-15 19:02:51 -0700630 var executable bool
Dan Willemsen017d8932016-08-04 15:43:03 -0700631
Colin Cross09f11052018-09-21 15:12:39 -0700632 var s os.FileInfo
633 var err error
634 if z.followSymlinks {
635 s, err = z.fs.Stat(src)
636 } else {
637 s, err = z.fs.Lstat(src)
638 }
639
640 if err != nil {
Colin Cross4be8f9e2018-09-28 15:16:48 -0700641 if os.IsNotExist(err) && z.ignoreMissingFiles {
642 fmt.Fprintln(z.stderr, "warning:", err)
643 return nil
644 }
Dan Willemsena59a3bc2016-08-03 17:47:23 -0700645 return err
Colin Cross9cb51db2019-06-17 14:12:41 -0700646 }
647
648 createParentDirs := func(dest, src string) error {
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700649 if err := z.writeDirectory(filepath.Dir(dest), src, emulateJar); err != nil {
Colin Crosse5580972017-08-30 17:40:21 -0700650 return err
651 }
652
653 if prev, exists := z.createdDirs[dest]; exists {
654 return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src)
655 }
Colin Cross7ddd08a2022-08-15 15:47:41 -0700656
657 return nil
658 }
659
660 checkDuplicateFiles := func(dest, src string) (bool, error) {
Colin Crosse5580972017-08-30 17:40:21 -0700661 if prev, exists := z.createdFiles[dest]; exists {
Colin Cross7ddd08a2022-08-15 15:47:41 -0700662 if prev != src {
663 return true, ConflictingFileError{
664 Dest: dest,
665 Prev: prev,
666 Src: src,
667 }
668 }
669 return true, nil
Colin Crosse5580972017-08-30 17:40:21 -0700670 }
671
672 z.createdFiles[dest] = src
Colin Cross7ddd08a2022-08-15 15:47:41 -0700673 return false, nil
Colin Cross9cb51db2019-06-17 14:12:41 -0700674 }
675
676 if s.IsDir() {
677 if z.directories {
678 return z.writeDirectory(dest, src, emulateJar)
679 }
680 return nil
681 } else if s.Mode()&os.ModeSymlink != 0 {
682 err = createParentDirs(dest, src)
683 if err != nil {
684 return err
685 }
686
Colin Cross7ddd08a2022-08-15 15:47:41 -0700687 duplicate, err := checkDuplicateFiles(dest, src)
688 if err != nil {
689 return err
690 }
691 if duplicate {
692 return nil
693 }
694
Colin Cross9cb51db2019-06-17 14:12:41 -0700695 return z.writeSymlink(dest, src)
696 } else if s.Mode().IsRegular() {
697 r, err := z.fs.Open(src)
698 if err != nil {
699 return err
700 }
701
702 if srcJar && filepath.Ext(src) == ".java" {
703 // rewrite the destination using the package path if it can be determined
704 pkg, err := jar.JavaPackage(r, src)
705 if err != nil {
706 // ignore errors for now, leaving the file at in its original location in the zip
707 } else {
708 dest = filepath.Join(filepath.Join(strings.Split(pkg, ".")...), filepath.Base(src))
709 }
710
711 _, err = r.Seek(0, io.SeekStart)
712 if err != nil {
713 return err
714 }
Colin Crosse5580972017-08-30 17:40:21 -0700715 }
716
Dan Willemsen017d8932016-08-04 15:43:03 -0700717 fileSize = s.Size()
Dan Willemsen10462b32017-03-15 19:02:51 -0700718 executable = s.Mode()&0100 != 0
Colin Cross957cc4e2015-04-24 15:10:32 -0700719
Colin Cross9cb51db2019-06-17 14:12:41 -0700720 header := &zip.FileHeader{
721 Name: dest,
722 Method: method,
723 UncompressedSize64: uint64(fileSize),
724 }
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700725
Chris Grossfa5b4e92021-06-02 12:56:08 -0700726 mode := os.FileMode(0644)
Colin Cross9cb51db2019-06-17 14:12:41 -0700727 if executable {
Chris Grossfa5b4e92021-06-02 12:56:08 -0700728 mode = 0755
Colin Cross9cb51db2019-06-17 14:12:41 -0700729 }
Colin Crosse070c3a2021-05-11 16:03:57 -0700730 header.SetMode(mode)
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700731
Colin Cross9cb51db2019-06-17 14:12:41 -0700732 err = createParentDirs(dest, src)
733 if err != nil {
734 return err
735 }
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700736
Colin Cross7ddd08a2022-08-15 15:47:41 -0700737 duplicate, err := checkDuplicateFiles(dest, src)
738 if err != nil {
739 return err
740 }
741 if duplicate {
742 return nil
743 }
744
Colin Cross9cb51db2019-06-17 14:12:41 -0700745 return z.writeFileContents(header, r)
746 } else {
747 return fmt.Errorf("%s is not a file, directory, or symlink", src)
748 }
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700749}
750
Sasha Smundak8eedba62020-11-16 19:00:27 -0800751func (z *ZipWriter) addManifest(dest string, src string, _ uint16) error {
Colin Crosse5580972017-08-30 17:40:21 -0700752 if prev, exists := z.createdDirs[dest]; exists {
753 return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src)
754 }
755 if prev, exists := z.createdFiles[dest]; exists {
Colin Cross7ddd08a2022-08-15 15:47:41 -0700756 if prev != src {
757 return ConflictingFileError{
758 Dest: dest,
759 Prev: prev,
760 Src: src,
761 }
762 }
763 return nil
Colin Crosse5580972017-08-30 17:40:21 -0700764 }
765
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700766 if err := z.writeDirectory(filepath.Dir(dest), src, true); err != nil {
Colin Cross635acc92017-09-12 22:50:46 -0700767 return err
Jeff Gastoncef50b92017-08-23 15:41:35 -0700768 }
769
Colin Cross05518bc2018-09-27 15:06:19 -0700770 var contents []byte
771 if src != "" {
772 f, err := z.fs.Open(src)
773 if err != nil {
774 return err
775 }
776
777 contents, err = ioutil.ReadAll(f)
778 f.Close()
779 if err != nil {
780 return err
781 }
782 }
783
784 fh, buf, err := jar.ManifestFileContents(contents)
Colin Cross635acc92017-09-12 22:50:46 -0700785 if err != nil {
786 return err
Jeff Gastoncef50b92017-08-23 15:41:35 -0700787 }
788
Colin Cross635acc92017-09-12 22:50:46 -0700789 reader := &byteReaderCloser{bytes.NewReader(buf), ioutil.NopCloser(nil)}
790
791 return z.writeFileContents(fh, reader)
Jeff Gastoncef50b92017-08-23 15:41:35 -0700792}
793
Colin Cross05518bc2018-09-27 15:06:19 -0700794func (z *ZipWriter) writeFileContents(header *zip.FileHeader, r pathtools.ReaderAtSeekerCloser) (err error) {
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700795
796 header.SetModTime(z.time)
797
Dan Willemsen017d8932016-08-04 15:43:03 -0700798 compressChan := make(chan *zipEntry, 1)
799 z.writeOps <- compressChan
800
801 // Pre-fill a zipEntry, it will be sent in the compressChan once
802 // we're sure about the Method and CRC.
803 ze := &zipEntry{
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700804 fh: header,
Dan Willemsen10462b32017-03-15 19:02:51 -0700805 }
Dan Willemsen017d8932016-08-04 15:43:03 -0700806
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700807 ze.allocatedSize = int64(header.UncompressedSize64)
Jeff Gaston175f34c2017-08-17 21:43:21 -0700808 z.cpuRateLimiter.Request()
809 z.memoryRateLimiter.Request(ze.allocatedSize)
Dan Willemsen017d8932016-08-04 15:43:03 -0700810
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700811 fileSize := int64(header.UncompressedSize64)
812 if fileSize == 0 {
813 fileSize = int64(header.UncompressedSize)
814 }
815
816 if header.Method == zip.Deflate && fileSize >= minParallelFileSize {
Dan Willemsen017d8932016-08-04 15:43:03 -0700817 wg := new(sync.WaitGroup)
818
819 // Allocate enough buffer to hold all readers. We'll limit
820 // this based on actual buffer sizes in RateLimit.
821 ze.futureReaders = make(chan chan io.Reader, (fileSize/parallelBlockSize)+1)
822
Zhenhuang Wangb8451b82023-01-06 20:58:01 +0800823 // Calculate the CRC and SHA256 in the background, since reading
824 // the entire file could take a while.
Dan Willemsen017d8932016-08-04 15:43:03 -0700825 //
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700826 // We could split this up into chunks as well, but it's faster
Dan Willemsen017d8932016-08-04 15:43:03 -0700827 // than the compression. Due to the Go Zip API, we also need to
828 // know the result before we can begin writing the compressed
829 // data out to the zipfile.
Zhenhuang Wangb8451b82023-01-06 20:58:01 +0800830 //
831 // We calculate SHA256 only if `-sha256` is set.
Dan Willemsen017d8932016-08-04 15:43:03 -0700832 wg.Add(1)
Zhenhuang Wangb8451b82023-01-06 20:58:01 +0800833 go z.checksumFileAsync(r, ze, compressChan, wg)
Dan Willemsen017d8932016-08-04 15:43:03 -0700834
835 for start := int64(0); start < fileSize; start += parallelBlockSize {
836 sr := io.NewSectionReader(r, start, parallelBlockSize)
837 resultChan := make(chan io.Reader, 1)
838 ze.futureReaders <- resultChan
839
Jeff Gaston175f34c2017-08-17 21:43:21 -0700840 z.cpuRateLimiter.Request()
Dan Willemsen017d8932016-08-04 15:43:03 -0700841
842 last := !(start+parallelBlockSize < fileSize)
843 var dict []byte
844 if start >= windowSize {
845 dict, err = ioutil.ReadAll(io.NewSectionReader(r, start-windowSize, windowSize))
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700846 if err != nil {
847 return err
848 }
Dan Willemsen017d8932016-08-04 15:43:03 -0700849 }
850
851 wg.Add(1)
Jeff Gaston175f34c2017-08-17 21:43:21 -0700852 go z.compressPartialFile(sr, dict, last, resultChan, wg)
Dan Willemsen017d8932016-08-04 15:43:03 -0700853 }
854
855 close(ze.futureReaders)
856
857 // Close the file handle after all readers are done
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700858 go func(wg *sync.WaitGroup, closer io.Closer) {
Dan Willemsen017d8932016-08-04 15:43:03 -0700859 wg.Wait()
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700860 closer.Close()
Dan Willemsen017d8932016-08-04 15:43:03 -0700861 }(wg, r)
862 } else {
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700863 go func() {
864 z.compressWholeFile(ze, r, compressChan)
865 r.Close()
866 }()
Dan Willemsen017d8932016-08-04 15:43:03 -0700867 }
868
869 return nil
870}
871
Zhenhuang Wangb8451b82023-01-06 20:58:01 +0800872func (z *ZipWriter) checksumFileAsync(r io.ReadSeeker, ze *zipEntry, resultChan chan *zipEntry, wg *sync.WaitGroup) {
Dan Willemsen017d8932016-08-04 15:43:03 -0700873 defer wg.Done()
Jeff Gaston175f34c2017-08-17 21:43:21 -0700874 defer z.cpuRateLimiter.Finish()
Dan Willemsen017d8932016-08-04 15:43:03 -0700875
Zhenhuang Wangb8451b82023-01-06 20:58:01 +0800876 z.checksumFile(r, ze)
877
878 resultChan <- ze
879 close(resultChan)
880}
881
882func (z *ZipWriter) checksumFile(r io.ReadSeeker, ze *zipEntry) {
Dan Willemsen017d8932016-08-04 15:43:03 -0700883 crc := crc32.NewIEEE()
Zhenhuang Wangb8451b82023-01-06 20:58:01 +0800884 writers := []io.Writer{crc}
885
886 var shaHasher hash.Hash
887 if z.sha256Checksum && !ze.fh.Mode().IsDir() {
888 shaHasher = sha256.New()
889 writers = append(writers, shaHasher)
890 }
891
892 w := io.MultiWriter(writers...)
893
894 _, err := io.Copy(w, r)
Dan Willemsen017d8932016-08-04 15:43:03 -0700895 if err != nil {
896 z.errors <- err
897 return
898 }
899
900 ze.fh.CRC32 = crc.Sum32()
Zhenhuang Wangb8451b82023-01-06 20:58:01 +0800901 if shaHasher != nil {
902 z.appendSHAToExtra(ze, shaHasher.Sum(nil))
903 }
904}
905
906func (z *ZipWriter) appendSHAToExtra(ze *zipEntry, checksum []byte) {
907 // The block of SHA256 checksum consist of:
908 // - Header ID, equals to Sha256HeaderID (2 bytes)
909 // - Data size (2 bytes)
910 // - Data block:
911 // - Signature, equals to Sha256HeaderSignature (2 bytes)
912 // - Data, SHA checksum value
913 var buf []byte
914 buf = binary.LittleEndian.AppendUint16(buf, Sha256HeaderID)
915 buf = binary.LittleEndian.AppendUint16(buf, uint16(len(checksum)+2))
916 buf = binary.LittleEndian.AppendUint16(buf, Sha256HeaderSignature)
917 buf = append(buf, checksum...)
918 ze.fh.Extra = append(ze.fh.Extra, buf...)
Dan Willemsen017d8932016-08-04 15:43:03 -0700919}
920
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700921func (z *ZipWriter) compressPartialFile(r io.Reader, dict []byte, last bool, resultChan chan io.Reader, wg *sync.WaitGroup) {
Dan Willemsen017d8932016-08-04 15:43:03 -0700922 defer wg.Done()
923
924 result, err := z.compressBlock(r, dict, last)
925 if err != nil {
926 z.errors <- err
927 return
928 }
929
Jeff Gaston175f34c2017-08-17 21:43:21 -0700930 z.cpuRateLimiter.Finish()
931
Dan Willemsen017d8932016-08-04 15:43:03 -0700932 resultChan <- result
933}
934
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700935func (z *ZipWriter) compressBlock(r io.Reader, dict []byte, last bool) (*bytes.Buffer, error) {
Dan Willemsen017d8932016-08-04 15:43:03 -0700936 buf := new(bytes.Buffer)
937 var fw *flate.Writer
938 var err error
939 if len(dict) > 0 {
940 // There's no way to Reset a Writer with a new dictionary, so
941 // don't use the Pool
942 fw, err = flate.NewWriterDict(buf, z.compLevel, dict)
943 } else {
944 var ok bool
945 if fw, ok = z.compressorPool.Get().(*flate.Writer); ok {
946 fw.Reset(buf)
947 } else {
948 fw, err = flate.NewWriter(buf, z.compLevel)
949 }
950 defer z.compressorPool.Put(fw)
951 }
952 if err != nil {
953 return nil, err
954 }
955
956 _, err = io.Copy(fw, r)
957 if err != nil {
958 return nil, err
959 }
960 if last {
961 fw.Close()
962 } else {
963 fw.Flush()
964 }
965
966 return buf, nil
967}
968
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700969func (z *ZipWriter) compressWholeFile(ze *zipEntry, r io.ReadSeeker, compressChan chan *zipEntry) {
Zhenhuang Wangb8451b82023-01-06 20:58:01 +0800970 z.checksumFile(r, ze)
Dan Willemsen017d8932016-08-04 15:43:03 -0700971
Zhenhuang Wangb8451b82023-01-06 20:58:01 +0800972 _, err := r.Seek(0, 0)
Colin Cross2fe66872015-03-30 17:20:39 -0700973 if err != nil {
Dan Willemsen017d8932016-08-04 15:43:03 -0700974 z.errors <- err
975 return
Colin Cross2fe66872015-03-30 17:20:39 -0700976 }
977
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700978 readFile := func(reader io.ReadSeeker) ([]byte, error) {
979 _, err := reader.Seek(0, 0)
Nan Zhangf281bd82017-04-25 16:47:45 -0700980 if err != nil {
981 return nil, err
982 }
983
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700984 buf, err := ioutil.ReadAll(reader)
Nan Zhangf281bd82017-04-25 16:47:45 -0700985 if err != nil {
986 return nil, err
987 }
988
989 return buf, nil
990 }
Dan Willemsen017d8932016-08-04 15:43:03 -0700991
Dan Willemsena8b55022017-03-15 21:49:26 -0700992 ze.futureReaders = make(chan chan io.Reader, 1)
Dan Willemsen017d8932016-08-04 15:43:03 -0700993 futureReader := make(chan io.Reader, 1)
994 ze.futureReaders <- futureReader
995 close(ze.futureReaders)
996
Nan Zhangf281bd82017-04-25 16:47:45 -0700997 if ze.fh.Method == zip.Deflate {
998 compressed, err := z.compressBlock(r, nil, true)
999 if err != nil {
1000 z.errors <- err
1001 return
1002 }
1003 if uint64(compressed.Len()) < ze.fh.UncompressedSize64 {
1004 futureReader <- compressed
Nan Zhangf281bd82017-04-25 16:47:45 -07001005 } else {
1006 buf, err := readFile(r)
1007 if err != nil {
1008 z.errors <- err
1009 return
1010 }
1011 ze.fh.Method = zip.Store
1012 futureReader <- bytes.NewReader(buf)
Nan Zhangf281bd82017-04-25 16:47:45 -07001013 }
Dan Willemsen017d8932016-08-04 15:43:03 -07001014 } else {
Nan Zhangf281bd82017-04-25 16:47:45 -07001015 buf, err := readFile(r)
Dan Willemsen017d8932016-08-04 15:43:03 -07001016 if err != nil {
1017 z.errors <- err
1018 return
1019 }
Dan Willemsen017d8932016-08-04 15:43:03 -07001020 ze.fh.Method = zip.Store
1021 futureReader <- bytes.NewReader(buf)
Dan Willemsen017d8932016-08-04 15:43:03 -07001022 }
Nan Zhangf281bd82017-04-25 16:47:45 -07001023
Jeff Gaston175f34c2017-08-17 21:43:21 -07001024 z.cpuRateLimiter.Finish()
1025
Dan Willemsen017d8932016-08-04 15:43:03 -07001026 close(futureReader)
1027
1028 compressChan <- ze
1029 close(compressChan)
Colin Cross2fe66872015-03-30 17:20:39 -07001030}
Colin Crosse19c7932015-04-24 15:08:38 -07001031
Colin Crosse5580972017-08-30 17:40:21 -07001032// writeDirectory annotates that dir is a directory created for the src file or directory, and adds
1033// the directory entry to the zip file if directories are enabled.
Jeff Gastonc3bdc972017-10-12 12:18:19 -07001034func (z *ZipWriter) writeDirectory(dir string, src string, emulateJar bool) error {
Jeff Gaston2d174132017-08-15 18:05:56 -07001035 // clean the input
Colin Crosse5580972017-08-30 17:40:21 -07001036 dir = filepath.Clean(dir)
Jeff Gaston2d174132017-08-15 18:05:56 -07001037
1038 // discover any uncreated directories in the path
Sasha Smundak8eedba62020-11-16 19:00:27 -08001039 var zipDirs []string
Colin Crosse5580972017-08-30 17:40:21 -07001040 for dir != "" && dir != "." {
1041 if _, exists := z.createdDirs[dir]; exists {
1042 break
1043 }
Jeff Gaston2d174132017-08-15 18:05:56 -07001044
Colin Crosse5580972017-08-30 17:40:21 -07001045 if prev, exists := z.createdFiles[dir]; exists {
1046 return fmt.Errorf("destination %q is both a directory %q and a file %q", dir, src, prev)
1047 }
1048
1049 z.createdDirs[dir] = src
Jeff Gaston2d174132017-08-15 18:05:56 -07001050 // parent directories precede their children
Colin Crosse5580972017-08-30 17:40:21 -07001051 zipDirs = append([]string{dir}, zipDirs...)
Jeff Gaston2d174132017-08-15 18:05:56 -07001052
Colin Crosse5580972017-08-30 17:40:21 -07001053 dir = filepath.Dir(dir)
Dan Willemsena59a3bc2016-08-03 17:47:23 -07001054 }
1055
Colin Crosse5580972017-08-30 17:40:21 -07001056 if z.directories {
1057 // make a directory entry for each uncreated directory
1058 for _, cleanDir := range zipDirs {
Colin Cross635acc92017-09-12 22:50:46 -07001059 var dirHeader *zip.FileHeader
Colin Crosse19c7932015-04-24 15:08:38 -07001060
Jeff Gastonc3bdc972017-10-12 12:18:19 -07001061 if emulateJar && cleanDir+"/" == jar.MetaDir {
Colin Cross635acc92017-09-12 22:50:46 -07001062 dirHeader = jar.MetaDirFileHeader()
1063 } else {
1064 dirHeader = &zip.FileHeader{
1065 Name: cleanDir + "/",
1066 }
Chris Grossfa5b4e92021-06-02 12:56:08 -07001067 dirHeader.SetMode(0755 | os.ModeDir)
Colin Crosse5580972017-08-30 17:40:21 -07001068 }
Jeff Gaston8edbb3a2017-08-22 20:05:28 -07001069
Colin Cross635acc92017-09-12 22:50:46 -07001070 dirHeader.SetModTime(z.time)
1071
Colin Crosse5580972017-08-30 17:40:21 -07001072 ze := make(chan *zipEntry, 1)
1073 ze <- &zipEntry{
1074 fh: dirHeader,
1075 }
1076 close(ze)
1077 z.writeOps <- ze
Colin Crosse19c7932015-04-24 15:08:38 -07001078 }
Colin Crosse19c7932015-04-24 15:08:38 -07001079 }
1080
1081 return nil
1082}
Dan Willemsena59a3bc2016-08-03 17:47:23 -07001083
Jeff Gastonc3bdc972017-10-12 12:18:19 -07001084func (z *ZipWriter) writeSymlink(rel, file string) error {
Dan Willemsena59a3bc2016-08-03 17:47:23 -07001085 fileHeader := &zip.FileHeader{
1086 Name: rel,
1087 }
1088 fileHeader.SetModTime(z.time)
Colin Cross297d9bc2018-06-22 16:37:47 -07001089 fileHeader.SetMode(0777 | os.ModeSymlink)
Dan Willemsena59a3bc2016-08-03 17:47:23 -07001090
Colin Cross05518bc2018-09-27 15:06:19 -07001091 dest, err := z.fs.Readlink(file)
Dan Willemsena59a3bc2016-08-03 17:47:23 -07001092 if err != nil {
1093 return err
1094 }
1095
Colin Cross297d9bc2018-06-22 16:37:47 -07001096 fileHeader.UncompressedSize64 = uint64(len(dest))
1097 fileHeader.CRC32 = crc32.ChecksumIEEE([]byte(dest))
1098
Dan Willemsen017d8932016-08-04 15:43:03 -07001099 ze := make(chan *zipEntry, 1)
1100 futureReaders := make(chan chan io.Reader, 1)
1101 futureReader := make(chan io.Reader, 1)
1102 futureReaders <- futureReader
1103 close(futureReaders)
1104 futureReader <- bytes.NewBufferString(dest)
1105 close(futureReader)
1106
Dan Willemsen017d8932016-08-04 15:43:03 -07001107 ze <- &zipEntry{
1108 fh: fileHeader,
1109 futureReaders: futureReaders,
1110 }
1111 close(ze)
1112 z.writeOps <- ze
1113
1114 return nil
Dan Willemsena59a3bc2016-08-03 17:47:23 -07001115}