mirror of
https://github.com/servo/servo.git
synced 2025-08-06 06:00:15 +01:00
WebIDL codegen: Replace cmake with a single Python script
When playing around with Cargo’s new timing visualization: https://internals.rust-lang.org/t/exploring-crate-graph-build-times-with-cargo-build-ztimings/10975/21 … I was surprised to see the `script` crate’s build script take 76 seconds. I did not expect WebIDL bindings generation to be *that* computationally intensive. It turns out almost all of this time is overhead. The build script uses CMake to generate bindings for each WebIDL file in parallel, but that causes a lot of work to be repeated 366 times: * Starting up a Python VM * Importing (parts of) the Python standard library * Importing ~16k lines of our Python code * Recompiling the latter to bytecode, since we used `python -B` to disable writing `.pyc` file * Deserializing with `cPickle` and recreating in memory the results of parsing all WebIDL files ---- This commit remove the use of CMake and cPickle for the `script` crate. Instead, all WebIDL bindings generation is done sequentially in a single Python process. This takes 2 to 3 seconds.
This commit is contained in:
Simon Sapin
parent
049527872e
commit
5c60023cb8
9 changed files with 177 additions and 457 deletions
|
|
@ -1,54 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
import sys
|
||||
import os
|
||||
sys.path.append(os.path.join(".", "parser"))
|
||||
sys.path.append(os.path.join(".", "ply"))
|
||||
import cPickle
|
||||
from Configuration import Configuration
|
||||
from CodegenRust import CGBindingRoot, replaceFileIfChanged
|
||||
|
||||
|
||||
def generate_binding_rs(config, outputprefix, webidlfile):
|
||||
"""
|
||||
|config| Is the configuration object.
|
||||
|outputprefix| is a prefix to use for the header guards and filename.
|
||||
"""
|
||||
|
||||
filename = outputprefix + ".rs"
|
||||
module = CGBindingRoot(config, outputprefix, webidlfile).define()
|
||||
if not module:
|
||||
print "Skipping empty module: %s" % (filename)
|
||||
elif replaceFileIfChanged(filename, module):
|
||||
print "Generating binding implementation: %s" % (filename)
|
||||
|
||||
|
||||
def main():
|
||||
# Parse arguments.
|
||||
from optparse import OptionParser
|
||||
usagestring = "usage: %prog configFile outputdir outputPrefix webIDLFile"
|
||||
o = OptionParser(usage=usagestring)
|
||||
(options, args) = o.parse_args()
|
||||
|
||||
if len(args) != 4:
|
||||
o.error(usagestring)
|
||||
configFile = os.path.normpath(args[0])
|
||||
outputdir = args[1]
|
||||
outputPrefix = args[2]
|
||||
webIDLFile = os.path.normpath(args[3])
|
||||
|
||||
# Load the parsing results
|
||||
resultsPath = os.path.join(outputdir, 'ParserResults.pkl')
|
||||
with open(resultsPath, 'rb') as f:
|
||||
parserData = cPickle.load(f)
|
||||
|
||||
# Create the configuration data.
|
||||
config = Configuration(configFile, parserData)
|
||||
|
||||
# Generate the prototype classes.
|
||||
generate_binding_rs(config, outputPrefix, webIDLFile)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -54,32 +54,6 @@ RUST_KEYWORDS = {"abstract", "alignof", "as", "become", "box", "break", "const",
|
|||
"use", "virtual", "where", "while", "yield"}
|
||||
|
||||
|
||||
def replaceFileIfChanged(filename, newContents):
|
||||
"""
|
||||
Read a copy of the old file, so that we don't touch it if it hasn't changed.
|
||||
Returns True if the file was updated, false otherwise.
|
||||
"""
|
||||
# XXXjdm This doesn't play well with make right now.
|
||||
# Force the file to always be updated, or else changing CodegenRust.py
|
||||
# will cause many autogenerated bindings to be regenerated perpetually
|
||||
# until the result is actually different.
|
||||
|
||||
# oldFileContents = ""
|
||||
# try:
|
||||
# with open(filename, 'rb') as oldFile:
|
||||
# oldFileContents = ''.join(oldFile.readlines())
|
||||
# except:
|
||||
# pass
|
||||
|
||||
# if newContents == oldFileContents:
|
||||
# return False
|
||||
|
||||
with open(filename, 'wb') as f:
|
||||
f.write(newContents)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def toStringBool(arg):
|
||||
return str(not not arg).lower()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,143 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
# We do one global pass over all the WebIDL to generate our prototype enum
|
||||
# and generate information for subsequent phases.
|
||||
|
||||
import sys
|
||||
import os
|
||||
import json
|
||||
sys.path.append(os.path.join(".", "parser"))
|
||||
sys.path.append(os.path.join(".", "ply"))
|
||||
import WebIDL
|
||||
import cPickle
|
||||
from Configuration import Configuration
|
||||
from CodegenRust import GlobalGenRoots, replaceFileIfChanged
|
||||
|
||||
|
||||
def generate_file(config, name, filename):
|
||||
root = getattr(GlobalGenRoots, name)(config)
|
||||
code = root.define()
|
||||
|
||||
if replaceFileIfChanged(filename, code):
|
||||
print "Generating %s" % (filename)
|
||||
else:
|
||||
print "%s hasn't changed - not touching it" % (filename)
|
||||
|
||||
|
||||
def main():
|
||||
# Parse arguments.
|
||||
from optparse import OptionParser
|
||||
usageString = "usage: %prog [options] configFile outputdir webidldir cssProperties.json docServoDir [files]"
|
||||
o = OptionParser(usage=usageString)
|
||||
o.add_option("--cachedir", dest='cachedir', default=None,
|
||||
help="Directory in which to cache lex/parse tables.")
|
||||
o.add_option("--filelist", dest='filelist', default=None,
|
||||
help="A file containing the list (one per line) of webidl files to process.")
|
||||
(options, args) = o.parse_args()
|
||||
|
||||
if len(args) < 2:
|
||||
o.error(usageString)
|
||||
|
||||
configFile = args[0]
|
||||
outputdir = args[1]
|
||||
baseDir = args[2]
|
||||
css_properties_json = args[3]
|
||||
doc_servo = args[4]
|
||||
if options.filelist is not None:
|
||||
fileList = [l.strip() for l in open(options.filelist).xreadlines()]
|
||||
else:
|
||||
fileList = args[3:]
|
||||
|
||||
# Parse the WebIDL.
|
||||
parser = WebIDL.Parser(options.cachedir)
|
||||
for filename in fileList:
|
||||
fullPath = os.path.normpath(os.path.join(baseDir, filename))
|
||||
with open(fullPath, 'rb') as f:
|
||||
lines = f.readlines()
|
||||
parser.parse(''.join(lines), fullPath)
|
||||
|
||||
add_css_properties_attributes(fileList, css_properties_json, parser)
|
||||
|
||||
parserResults = parser.finish()
|
||||
|
||||
# Write the parser results out to a pickle.
|
||||
resultsPath = os.path.join(outputdir, 'ParserResults.pkl')
|
||||
with open(resultsPath, 'wb') as resultsFile:
|
||||
cPickle.dump(parserResults, resultsFile, -1)
|
||||
|
||||
# Load the configuration.
|
||||
config = Configuration(configFile, parserResults)
|
||||
|
||||
to_generate = [
|
||||
('PrototypeList', 'PrototypeList.rs'),
|
||||
('RegisterBindings', 'RegisterBindings.rs'),
|
||||
('InterfaceObjectMap', 'InterfaceObjectMap.rs'),
|
||||
('InterfaceObjectMapData', 'InterfaceObjectMapData.json'),
|
||||
('InterfaceTypes', 'InterfaceTypes.rs'),
|
||||
('InheritTypes', 'InheritTypes.rs'),
|
||||
('Bindings', os.path.join('Bindings', 'mod.rs')),
|
||||
('UnionTypes', 'UnionTypes.rs'),
|
||||
]
|
||||
|
||||
for name, filename in to_generate:
|
||||
generate_file(config, name, os.path.join(outputdir, filename))
|
||||
|
||||
generate_file(config, 'SupportedDomApis', os.path.join(doc_servo, 'apis.html'))
|
||||
|
||||
|
||||
def add_css_properties_attributes(webidl_files, css_properties_json, parser):
|
||||
for filename in webidl_files:
|
||||
if os.path.basename(filename) == "CSSStyleDeclaration.webidl":
|
||||
break
|
||||
else:
|
||||
return
|
||||
|
||||
css_properties = json.load(open(css_properties_json, "rb"))
|
||||
idl = "partial interface CSSStyleDeclaration {\n%s\n};\n" % "\n".join(
|
||||
" [%sCEReactions, SetterThrows] attribute [TreatNullAs=EmptyString] DOMString %s;" % (
|
||||
('Pref="%s", ' % data["pref"] if data["pref"] else ""),
|
||||
attribute_name
|
||||
)
|
||||
for (kind, properties_list) in sorted(css_properties.items())
|
||||
for (property_name, data) in sorted(properties_list.items())
|
||||
for attribute_name in attribute_names(property_name)
|
||||
)
|
||||
parser.parse(idl.encode("utf-8"), "CSSStyleDeclaration_generated.webidl")
|
||||
|
||||
|
||||
def attribute_names(property_name):
|
||||
# https://drafts.csswg.org/cssom/#dom-cssstyledeclaration-dashed-attribute
|
||||
if property_name != "float":
|
||||
yield property_name
|
||||
else:
|
||||
yield "_float"
|
||||
|
||||
# https://drafts.csswg.org/cssom/#dom-cssstyledeclaration-camel-cased-attribute
|
||||
if "-" in property_name:
|
||||
yield "".join(camel_case(property_name))
|
||||
|
||||
# https://drafts.csswg.org/cssom/#dom-cssstyledeclaration-webkit-cased-attribute
|
||||
if property_name.startswith("-webkit-"):
|
||||
yield "".join(camel_case(property_name), True)
|
||||
|
||||
|
||||
# https://drafts.csswg.org/cssom/#css-property-to-idl-attribute
|
||||
def camel_case(chars, webkit_prefixed=False):
|
||||
if webkit_prefixed:
|
||||
chars = chars[1:]
|
||||
next_is_uppercase = False
|
||||
for c in chars:
|
||||
if c == '-':
|
||||
next_is_uppercase = True
|
||||
elif next_is_uppercase:
|
||||
next_is_uppercase = False
|
||||
# Should be ASCII-uppercase, but all non-custom CSS property names are within ASCII
|
||||
yield c.upper()
|
||||
else:
|
||||
yield c
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
"""
|
||||
Run a python script, adding extra directories to the python path.
|
||||
"""
|
||||
|
||||
|
||||
def main(args):
|
||||
def usage():
|
||||
print >>sys.stderr, "pythonpath.py -I directory script.py [args...]"
|
||||
sys.exit(150)
|
||||
|
||||
paths = []
|
||||
|
||||
while True:
|
||||
try:
|
||||
arg = args[0]
|
||||
except IndexError:
|
||||
usage()
|
||||
|
||||
if arg == '-I':
|
||||
args.pop(0)
|
||||
try:
|
||||
path = args.pop(0)
|
||||
except IndexError:
|
||||
usage()
|
||||
|
||||
paths.append(os.path.abspath(path))
|
||||
continue
|
||||
|
||||
if arg.startswith('-I'):
|
||||
paths.append(os.path.abspath(args.pop(0)[2:]))
|
||||
continue
|
||||
|
||||
if arg.startswith('-D'):
|
||||
os.chdir(args.pop(0)[2:])
|
||||
continue
|
||||
|
||||
break
|
||||
|
||||
script = args[0]
|
||||
|
||||
sys.path[0:0] = [os.path.abspath(os.path.dirname(script))] + paths
|
||||
sys.argv = args
|
||||
sys.argc = len(args)
|
||||
|
||||
frozenglobals['__name__'] = '__main__'
|
||||
frozenglobals['__file__'] = script
|
||||
|
||||
execfile(script, frozenglobals)
|
||||
|
||||
# Freeze scope here ... why this makes things work I have no idea ...
|
||||
frozenglobals = globals()
|
||||
|
||||
import sys
|
||||
import os
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(sys.argv[1:])
|
||||
119
components/script/dom/bindings/codegen/run.py
Normal file
119
components/script/dom/bindings/codegen/run.py
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
|
||||
|
||||
def main():
|
||||
os.chdir(os.path.join(os.path.dirname(__file__)))
|
||||
sys.path[0:0] = ["./parser", "./ply"]
|
||||
|
||||
css_properties_json, out_dir, doc_servo = sys.argv[1:]
|
||||
webidls_dir = "../../webidls"
|
||||
config_file = "Bindings.conf"
|
||||
|
||||
import WebIDL
|
||||
from Configuration import Configuration
|
||||
from CodegenRust import CGBindingRoot
|
||||
|
||||
parser = WebIDL.Parser(make_dir(out_dir, "cache"))
|
||||
webidls = [name for name in os.listdir(webidls_dir) if name.endswith(".webidl")]
|
||||
for webidl in webidls:
|
||||
filename = os.path.join(webidls_dir, webidl)
|
||||
with open(filename, "rb") as f:
|
||||
parser.parse(f.read(), filename)
|
||||
|
||||
add_css_properties_attributes(css_properties_json, parser)
|
||||
parser_results = parser.finish()
|
||||
config = Configuration(config_file, parser_results)
|
||||
make_dir(out_dir, "Bindings")
|
||||
|
||||
for name, filename in [
|
||||
("PrototypeList", "PrototypeList.rs"),
|
||||
("RegisterBindings", "RegisterBindings.rs"),
|
||||
("InterfaceObjectMap", "InterfaceObjectMap.rs"),
|
||||
("InterfaceObjectMapData", "InterfaceObjectMapData.json"),
|
||||
("InterfaceTypes", "InterfaceTypes.rs"),
|
||||
("InheritTypes", "InheritTypes.rs"),
|
||||
("Bindings", "Bindings/mod.rs"),
|
||||
("UnionTypes", "UnionTypes.rs"),
|
||||
]:
|
||||
generate(config, name, os.path.join(out_dir, filename))
|
||||
make_dir(doc_servo)
|
||||
generate(config, "SupportedDomApis", os.path.join(doc_servo, "apis.html"))
|
||||
|
||||
for webidl in webidls:
|
||||
filename = os.path.join(webidls_dir, webidl)
|
||||
prefix = "Bindings/%sBinding" % webidl[:-len(".webidl")]
|
||||
module = CGBindingRoot(config, prefix, filename).define()
|
||||
if module:
|
||||
with open(os.path.join(out_dir, prefix + ".rs"), "wb") as f:
|
||||
f.write(module)
|
||||
|
||||
|
||||
def make_dir(out_dir, nested=""):
|
||||
path = os.path.join(out_dir, nested)
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
return path
|
||||
|
||||
|
||||
def generate(config, name, filename):
|
||||
from CodegenRust import GlobalGenRoots
|
||||
root = getattr(GlobalGenRoots, name)(config)
|
||||
code = root.define()
|
||||
with open(filename, "wb") as f:
|
||||
f.write(code)
|
||||
|
||||
|
||||
def add_css_properties_attributes(css_properties_json, parser):
|
||||
css_properties = json.load(open(css_properties_json, "rb"))
|
||||
idl = "partial interface CSSStyleDeclaration {\n%s\n};\n" % "\n".join(
|
||||
" [%sCEReactions, SetterThrows] attribute [TreatNullAs=EmptyString] DOMString %s;" % (
|
||||
('Pref="%s", ' % data["pref"] if data["pref"] else ""),
|
||||
attribute_name
|
||||
)
|
||||
for (kind, properties_list) in sorted(css_properties.items())
|
||||
for (property_name, data) in sorted(properties_list.items())
|
||||
for attribute_name in attribute_names(property_name)
|
||||
)
|
||||
parser.parse(idl.encode("utf-8"), "CSSStyleDeclaration_generated.webidl")
|
||||
|
||||
|
||||
def attribute_names(property_name):
|
||||
# https://drafts.csswg.org/cssom/#dom-cssstyledeclaration-dashed-attribute
|
||||
if property_name != "float":
|
||||
yield property_name
|
||||
else:
|
||||
yield "_float"
|
||||
|
||||
# https://drafts.csswg.org/cssom/#dom-cssstyledeclaration-camel-cased-attribute
|
||||
if "-" in property_name:
|
||||
yield "".join(camel_case(property_name))
|
||||
|
||||
# https://drafts.csswg.org/cssom/#dom-cssstyledeclaration-webkit-cased-attribute
|
||||
if property_name.startswith("-webkit-"):
|
||||
yield "".join(camel_case(property_name), True)
|
||||
|
||||
|
||||
# https://drafts.csswg.org/cssom/#css-property-to-idl-attribute
|
||||
def camel_case(chars, webkit_prefixed=False):
|
||||
if webkit_prefixed:
|
||||
chars = chars[1:]
|
||||
next_is_uppercase = False
|
||||
for c in chars:
|
||||
if c == '-':
|
||||
next_is_uppercase = True
|
||||
elif next_is_uppercase:
|
||||
next_is_uppercase = False
|
||||
# Should be ASCII-uppercase, but all non-custom CSS property names are within ASCII
|
||||
yield c.upper()
|
||||
else:
|
||||
yield c
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue