[packager] Optimize archive accesses.
This commit is contained in:
parent
bfa9b4e82e
commit
eefdf718e1
|
@ -535,7 +535,7 @@ class Jarrer(FileRegistry, BaseFile):
|
||||||
dest = Dest(dest)
|
dest = Dest(dest)
|
||||||
assert isinstance(dest, Dest)
|
assert isinstance(dest, Dest)
|
||||||
|
|
||||||
from mozpack.mozjar import JarWriter, JarReader
|
from mozpack.mozjar import JarWriter, JarReader, JAR_BROTLI
|
||||||
try:
|
try:
|
||||||
old_jar = JarReader(fileobj=dest)
|
old_jar = JarReader(fileobj=dest)
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|
|
@ -6,6 +6,7 @@ from __future__ import absolute_import
|
||||||
|
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
import struct
|
import struct
|
||||||
|
import subprocess
|
||||||
import zlib
|
import zlib
|
||||||
import os
|
import os
|
||||||
from zipfile import (
|
from zipfile import (
|
||||||
|
@ -15,9 +16,11 @@ from zipfile import (
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
from urlparse import urlparse, ParseResult
|
from urlparse import urlparse, ParseResult
|
||||||
import mozpack.path as mozpath
|
import mozpack.path as mozpath
|
||||||
|
from mozbuild.util import memoize
|
||||||
|
|
||||||
JAR_STORED = ZIP_STORED
|
JAR_STORED = ZIP_STORED
|
||||||
JAR_DEFLATED = ZIP_DEFLATED
|
JAR_DEFLATED = ZIP_DEFLATED
|
||||||
|
JAR_BROTLI = 0x81
|
||||||
MAX_WBITS = 15
|
MAX_WBITS = 15
|
||||||
|
|
||||||
|
|
||||||
|
@ -262,13 +265,14 @@ class JarFileReader(object):
|
||||||
corresponding to the file in the jar archive, data a buffer containing
|
corresponding to the file in the jar archive, data a buffer containing
|
||||||
the file data.
|
the file data.
|
||||||
'''
|
'''
|
||||||
assert header['compression'] in [JAR_DEFLATED, JAR_STORED]
|
assert header['compression'] in [JAR_DEFLATED, JAR_STORED, JAR_BROTLI]
|
||||||
self._data = data
|
self._data = data
|
||||||
# Copy some local file header fields.
|
# Copy some local file header fields.
|
||||||
for name in ['filename', 'compressed_size',
|
for name in ['filename', 'compressed_size',
|
||||||
'uncompressed_size', 'crc32']:
|
'uncompressed_size', 'crc32']:
|
||||||
setattr(self, name, header[name])
|
setattr(self, name, header[name])
|
||||||
self.compressed = header['compression'] == JAR_DEFLATED
|
self.compressed = header['compression'] != JAR_STORED
|
||||||
|
self.compress = header['compression']
|
||||||
|
|
||||||
def read(self, length=-1):
|
def read(self, length=-1):
|
||||||
'''
|
'''
|
||||||
|
@ -317,7 +321,11 @@ class JarFileReader(object):
|
||||||
if hasattr(self, '_uncompressed_data'):
|
if hasattr(self, '_uncompressed_data'):
|
||||||
return self._uncompressed_data
|
return self._uncompressed_data
|
||||||
data = self.compressed_data
|
data = self.compressed_data
|
||||||
if self.compressed:
|
if self.compress == JAR_STORED:
|
||||||
|
data = data.tobytes()
|
||||||
|
elif self.compress == JAR_BROTLI:
|
||||||
|
data = Brotli.decompress(data.tobytes())
|
||||||
|
elif self.compress == JAR_DEFLATED:
|
||||||
data = zlib.decompress(data.tobytes(), -MAX_WBITS)
|
data = zlib.decompress(data.tobytes(), -MAX_WBITS)
|
||||||
else:
|
else:
|
||||||
data = data.tobytes()
|
data = data.tobytes()
|
||||||
|
@ -360,6 +368,13 @@ class JarReader(object):
|
||||||
'''
|
'''
|
||||||
del self._data
|
del self._data
|
||||||
|
|
||||||
|
@property
|
||||||
|
def compression(self):
|
||||||
|
entries = self.entries
|
||||||
|
if not entries:
|
||||||
|
return JAR_STORED
|
||||||
|
return max(f['compression'] for f in entries.itervalues())
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def entries(self):
|
def entries(self):
|
||||||
'''
|
'''
|
||||||
|
@ -473,6 +488,8 @@ class JarWriter(object):
|
||||||
self._data = fileobj
|
self._data = fileobj
|
||||||
else:
|
else:
|
||||||
self._data = open(file, 'wb')
|
self._data = open(file, 'wb')
|
||||||
|
if compress is True:
|
||||||
|
compress = JAR_DEFLATED
|
||||||
self._compress = compress
|
self._compress = compress
|
||||||
self._compress_level = compress_level
|
self._compress_level = compress_level
|
||||||
self._contents = OrderedDict()
|
self._contents = OrderedDict()
|
||||||
|
@ -574,12 +591,13 @@ class JarWriter(object):
|
||||||
'''
|
'''
|
||||||
Add a new member to the jar archive, with the given name and the given
|
Add a new member to the jar archive, with the given name and the given
|
||||||
data.
|
data.
|
||||||
The compress option indicates if the given data should be compressed
|
The compress option indicates how the given data should be compressed
|
||||||
(True), not compressed (False), or compressed according to the default
|
(one of JAR_STORED, JAR_DEFLATE or JAR_BROTLI), or compressed according
|
||||||
defined when creating the JarWriter (None).
|
to the default defined when creating the JarWriter (None). True and
|
||||||
When the data should be compressed (True or None with self.compress ==
|
False are allowed values for backwards compatibility, mapping,
|
||||||
True), it is only really compressed if the compressed size is smaller
|
respectively, to JAR_DEFLATE and JAR_STORED.
|
||||||
than the uncompressed size.
|
When the data should be compressed, it is only really compressed if
|
||||||
|
the compressed size is smaller than the uncompressed size.
|
||||||
The mode option gives the unix permissions that should be stored
|
The mode option gives the unix permissions that should be stored
|
||||||
for the jar entry.
|
for the jar entry.
|
||||||
If a duplicated member is found skip_duplicates will prevent raising
|
If a duplicated member is found skip_duplicates will prevent raising
|
||||||
|
@ -594,8 +612,12 @@ class JarWriter(object):
|
||||||
raise JarWriterError("File %s already in JarWriter" % name)
|
raise JarWriterError("File %s already in JarWriter" % name)
|
||||||
if compress is None:
|
if compress is None:
|
||||||
compress = self._compress
|
compress = self._compress
|
||||||
if (isinstance(data, JarFileReader) and data.compressed == compress) \
|
if compress is True:
|
||||||
or (isinstance(data, Deflater) and data.compress == compress):
|
compress = JAR_DEFLATED
|
||||||
|
if compress is False:
|
||||||
|
compress = JAR_STORED
|
||||||
|
if (isinstance(data, (JarFileReader, Deflater)) and \
|
||||||
|
data.compress == compress):
|
||||||
deflater = data
|
deflater = data
|
||||||
else:
|
else:
|
||||||
deflater = Deflater(compress, compress_level=self._compress_level)
|
deflater = Deflater(compress, compress_level=self._compress_level)
|
||||||
|
@ -619,7 +641,7 @@ class JarWriter(object):
|
||||||
if deflater.compressed:
|
if deflater.compressed:
|
||||||
entry['min_version'] = 20 # Version 2.0 supports deflated streams
|
entry['min_version'] = 20 # Version 2.0 supports deflated streams
|
||||||
entry['general_flag'] = 2 # Max compression
|
entry['general_flag'] = 2 # Max compression
|
||||||
entry['compression'] = JAR_DEFLATED
|
entry['compression'] = deflater.compress
|
||||||
else:
|
else:
|
||||||
entry['min_version'] = 10 # Version 1.0 for stored streams
|
entry['min_version'] = 10 # Version 1.0 for stored streams
|
||||||
entry['general_flag'] = 0
|
entry['general_flag'] = 0
|
||||||
|
@ -659,14 +681,21 @@ class Deflater(object):
|
||||||
'''
|
'''
|
||||||
def __init__(self, compress=True, compress_level=9):
|
def __init__(self, compress=True, compress_level=9):
|
||||||
'''
|
'''
|
||||||
Initialize a Deflater. The compress argument determines whether to
|
Initialize a Deflater. The compress argument determines how to
|
||||||
try to compress at all.
|
compress.
|
||||||
'''
|
'''
|
||||||
self._data = BytesIO()
|
self._data = BytesIO()
|
||||||
|
if compress is True:
|
||||||
|
compress = JAR_DEFLATED
|
||||||
|
elif compress is False:
|
||||||
|
compress = JAR_STORED
|
||||||
self.compress = compress
|
self.compress = compress
|
||||||
if compress:
|
if compress in (JAR_DEFLATED, JAR_BROTLI):
|
||||||
self._deflater = zlib.compressobj(compress_level, zlib.DEFLATED,
|
if compress == JAR_DEFLATED:
|
||||||
-MAX_WBITS)
|
self._deflater = zlib.compressobj(
|
||||||
|
compress_level, zlib.DEFLATED, -MAX_WBITS)
|
||||||
|
else:
|
||||||
|
self._deflater = BrotliCompress()
|
||||||
self._deflated = BytesIO()
|
self._deflated = BytesIO()
|
||||||
else:
|
else:
|
||||||
self._deflater = None
|
self._deflater = None
|
||||||
|
@ -759,6 +788,46 @@ class Deflater(object):
|
||||||
return self._data.getvalue()
|
return self._data.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
class Brotli(object):
|
||||||
|
@staticmethod
|
||||||
|
@memoize
|
||||||
|
def brotli_tool():
|
||||||
|
from buildconfig import topobjdir, substs
|
||||||
|
return os.path.join(topobjdir, 'dist', 'host', 'bin',
|
||||||
|
'brotli' + substs.get('BIN_SUFFIX', ''))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def run_brotli_tool(args, input):
|
||||||
|
proc = subprocess.Popen([Brotli.brotli_tool()] + args,
|
||||||
|
stdin=subprocess.PIPE,
|
||||||
|
stdout=subprocess.PIPE)
|
||||||
|
(stdout, _) = proc.communicate(input)
|
||||||
|
ret = proc.wait()
|
||||||
|
if ret != 0:
|
||||||
|
raise Exception("Brotli compression failed")
|
||||||
|
return stdout
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def compress(data):
|
||||||
|
return Brotli.run_brotli_tool(['--lgwin=17'], data)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def decompress(data):
|
||||||
|
return Brotli.run_brotli_tool(['--decompress'], data)
|
||||||
|
|
||||||
|
|
||||||
|
class BrotliCompress(object):
|
||||||
|
def __init__(self):
|
||||||
|
self._buf = BytesIO()
|
||||||
|
|
||||||
|
def compress(self, data):
|
||||||
|
self._buf.write(data)
|
||||||
|
return b''
|
||||||
|
|
||||||
|
def flush(self):
|
||||||
|
return Brotli.compress(self._buf.getvalue())
|
||||||
|
|
||||||
|
|
||||||
class JarLog(dict):
|
class JarLog(dict):
|
||||||
'''
|
'''
|
||||||
Helper to read the file Gecko generates when setting MOZ_JAR_LOG_FILE.
|
Helper to read the file Gecko generates when setting MOZ_JAR_LOG_FILE.
|
||||||
|
|
|
@ -37,6 +37,7 @@ from mozpack.chrome.manifest import (
|
||||||
Manifest,
|
Manifest,
|
||||||
)
|
)
|
||||||
from mozpack.errors import errors
|
from mozpack.errors import errors
|
||||||
|
from mozpack.mozjar import JAR_DEFLATED
|
||||||
from mozpack.packager.unpack import UnpackFinder
|
from mozpack.packager.unpack import UnpackFinder
|
||||||
from createprecomplete import generate_precomplete
|
from createprecomplete import generate_precomplete
|
||||||
|
|
||||||
|
@ -241,16 +242,17 @@ def repack(source, l10n, extra_l10n={}, non_resources=[], non_chrome=set()):
|
||||||
finders[base] = UnpackFinder(path)
|
finders[base] = UnpackFinder(path)
|
||||||
l10n_finder = ComposedFinder(finders)
|
l10n_finder = ComposedFinder(finders)
|
||||||
copier = FileCopier()
|
copier = FileCopier()
|
||||||
|
compress = min(app_finder.compressed, JAR_DEFLATED)
|
||||||
if app_finder.kind == 'flat':
|
if app_finder.kind == 'flat':
|
||||||
formatter = FlatFormatter(copier)
|
formatter = FlatFormatter(copier)
|
||||||
elif app_finder.kind == 'jar':
|
elif app_finder.kind == 'jar':
|
||||||
formatter = JarFormatter(copier,
|
formatter = JarFormatter(copier,
|
||||||
optimize=app_finder.optimizedjars,
|
optimize=app_finder.optimizedjars,
|
||||||
compress=app_finder.compressed)
|
compress=compress)
|
||||||
elif app_finder.kind == 'omni':
|
elif app_finder.kind == 'omni':
|
||||||
formatter = OmniJarFormatter(copier, app_finder.omnijar,
|
formatter = OmniJarFormatter(copier, app_finder.omnijar,
|
||||||
optimize=app_finder.optimizedjars,
|
optimize=app_finder.optimizedjars,
|
||||||
compress=app_finder.compressed,
|
compress=compress,
|
||||||
non_resources=non_resources)
|
non_resources=non_resources)
|
||||||
|
|
||||||
with errors.accumulate():
|
with errors.accumulate():
|
||||||
|
|
|
@ -54,7 +54,7 @@ class UnpackFinder(BaseFinder):
|
||||||
self.omnijar = None
|
self.omnijar = None
|
||||||
self.jarlogs = {}
|
self.jarlogs = {}
|
||||||
self.optimizedjars = False
|
self.optimizedjars = False
|
||||||
self.compressed = True
|
self.compressed = False
|
||||||
|
|
||||||
jars = set()
|
jars = set()
|
||||||
|
|
||||||
|
@ -146,8 +146,7 @@ class UnpackFinder(BaseFinder):
|
||||||
jar = JarReader(fileobj=file.open())
|
jar = JarReader(fileobj=file.open())
|
||||||
if jar.is_optimized:
|
if jar.is_optimized:
|
||||||
self.optimizedjars = True
|
self.optimizedjars = True
|
||||||
if not any(f.compressed for f in jar):
|
self.compressed = max(self.compressed, jar.compression)
|
||||||
self.compressed = False
|
|
||||||
if jar.last_preloaded:
|
if jar.last_preloaded:
|
||||||
jarlog = jar.entries.keys()
|
jarlog = jar.entries.keys()
|
||||||
self.jarlogs[path] = jarlog[:jarlog.index(jar.last_preloaded) + 1]
|
self.jarlogs[path] = jarlog[:jarlog.index(jar.last_preloaded) + 1]
|
||||||
|
|
|
@ -50,7 +50,7 @@ stage-package: $(MOZ_PKG_MANIFEST) $(MOZ_PKG_MANIFEST_DEPS)
|
||||||
) \
|
) \
|
||||||
$(if $(JARLOG_DIR),$(addprefix --jarlog ,$(wildcard $(JARLOG_FILE_AB_CD)))) \
|
$(if $(JARLOG_DIR),$(addprefix --jarlog ,$(wildcard $(JARLOG_FILE_AB_CD)))) \
|
||||||
$(if $(OPTIMIZEJARS),--optimizejars) \
|
$(if $(OPTIMIZEJARS),--optimizejars) \
|
||||||
$(if $(DISABLE_JAR_COMPRESSION),--disable-compression) \
|
$(addprefix --compress ,$(JAR_COMPRESSION)) \
|
||||||
$(addprefix --unify ,$(UNIFY_DIST)) \
|
$(addprefix --unify ,$(UNIFY_DIST)) \
|
||||||
$(MOZ_PKG_MANIFEST) '$(DIST)' '$(DIST)'/$(STAGEPATH)$(MOZ_PKG_DIR)$(if $(MOZ_PKG_MANIFEST),,$(_BINPATH)) \
|
$(MOZ_PKG_MANIFEST) '$(DIST)' '$(DIST)'/$(STAGEPATH)$(MOZ_PKG_DIR)$(if $(MOZ_PKG_MANIFEST),,$(_BINPATH)) \
|
||||||
$(if $(filter omni,$(MOZ_PACKAGER_FORMAT)),$(if $(NON_OMNIJAR_FILES),--non-resource $(NON_OMNIJAR_FILES)))
|
$(if $(filter omni,$(MOZ_PACKAGER_FORMAT)),$(if $(NON_OMNIJAR_FILES),--non-resource $(NON_OMNIJAR_FILES)))
|
||||||
|
|
|
@ -23,6 +23,7 @@ from mozpack.copier import (
|
||||||
Jarrer,
|
Jarrer,
|
||||||
)
|
)
|
||||||
from mozpack.errors import errors
|
from mozpack.errors import errors
|
||||||
|
from mozpack.mozjar import JAR_BROTLI
|
||||||
from mozpack.unify import UnifiedBuildFinder
|
from mozpack.unify import UnifiedBuildFinder
|
||||||
import mozpack.path as mozpath
|
import mozpack.path as mozpath
|
||||||
import buildconfig
|
import buildconfig
|
||||||
|
@ -270,9 +271,9 @@ def main():
|
||||||
help='Enable jar optimizations')
|
help='Enable jar optimizations')
|
||||||
parser.add_argument('--unify', default='',
|
parser.add_argument('--unify', default='',
|
||||||
help='Base directory of another build to unify with')
|
help='Base directory of another build to unify with')
|
||||||
parser.add_argument('--disable-compression', action='store_false',
|
parser.add_argument('--compress', choices=('none', 'deflate', 'brotli'),
|
||||||
dest='compress', default=True,
|
default='deflate',
|
||||||
help='Disable jar compression')
|
help='Use given jar compression (default: deflate)')
|
||||||
parser.add_argument('manifest', default=None, nargs='?',
|
parser.add_argument('manifest', default=None, nargs='?',
|
||||||
help='Manifest file name')
|
help='Manifest file name')
|
||||||
parser.add_argument('source', help='Source directory')
|
parser.add_argument('source', help='Source directory')
|
||||||
|
@ -290,15 +291,21 @@ def main():
|
||||||
for name, value in [split_define(d) for d in args.defines]:
|
for name, value in [split_define(d) for d in args.defines]:
|
||||||
defines[name] = value
|
defines[name] = value
|
||||||
|
|
||||||
|
compress = {
|
||||||
|
'none': False,
|
||||||
|
'deflate': True,
|
||||||
|
'brotli': JAR_BROTLI,
|
||||||
|
}[args.compress]
|
||||||
|
|
||||||
copier = FileCopier()
|
copier = FileCopier()
|
||||||
if args.format == 'flat':
|
if args.format == 'flat':
|
||||||
formatter = FlatFormatter(copier)
|
formatter = FlatFormatter(copier)
|
||||||
elif args.format == 'jar':
|
elif args.format == 'jar':
|
||||||
formatter = JarFormatter(copier, compress=args.compress, optimize=args.optimizejars)
|
formatter = JarFormatter(copier, compress=compress, optimize=args.optimizejars)
|
||||||
elif args.format == 'omni':
|
elif args.format == 'omni':
|
||||||
formatter = OmniJarFormatter(copier,
|
formatter = OmniJarFormatter(copier,
|
||||||
buildconfig.substs['OMNIJAR_NAME'],
|
buildconfig.substs['OMNIJAR_NAME'],
|
||||||
compress=args.compress,
|
compress=compress,
|
||||||
optimize=args.optimizejars,
|
optimize=args.optimizejars,
|
||||||
non_resources=args.non_resource)
|
non_resources=args.non_resource)
|
||||||
else:
|
else:
|
||||||
|
|
|
@ -385,7 +385,7 @@ ifneq (android,$(MOZ_WIDGET_TOOLKIT))
|
||||||
OPTIMIZEJARS = 1
|
OPTIMIZEJARS = 1
|
||||||
ifneq (gonk,$(MOZ_WIDGET_TOOLKIT))
|
ifneq (gonk,$(MOZ_WIDGET_TOOLKIT))
|
||||||
ifdef NIGHTLY_BUILD
|
ifdef NIGHTLY_BUILD
|
||||||
DISABLE_JAR_COMPRESSION = 1
|
JAR_COMPRESSION ?= none
|
||||||
endif
|
endif
|
||||||
endif
|
endif
|
||||||
endif
|
endif
|
||||||
|
|
Loading…
Reference in New Issue
Block a user