WebIDL codegen: Replace cmake with a single Python script

When playing around with Cargo’s new timing visualization:
https://internals.rust-lang.org/t/exploring-crate-graph-build-times-with-cargo-build-ztimings/10975/21

… I was surprised to see the `script` crate’s build script take 76 seconds.
I did not expect WebIDL bindings generation to be *that* computationally
intensive.

It turns out almost all of this time is overhead. The build script uses CMake
to generate bindings for each WebIDL file in parallel, but that causes a lot
of work to be repeated 366 times:

* Starting up a Python VM
* Importing (parts of) the Python standard library
* Importing ~16k lines of our Python code
* Recompiling the latter to bytecode, since we used `python -B` to disable
  writing `.pyc` file
* Deserializing with `cPickle` and recreating in memory the results
  of parsing all WebIDL files

----

This commit remove the use of CMake and cPickle for the `script` crate.
Instead, all WebIDL bindings generation is done sequentially
in a single Python process. This takes 2 to 3 seconds.
This commit is contained in:
Simon Sapin 2019-09-27 06:37:54 +02:00
parent 049527872e
commit 5c60023cb8
9 changed files with 177 additions and 457 deletions

View file

@ -1,104 +0,0 @@
project(script LANGUAGES)
cmake_minimum_required(VERSION 2.6)
set(DUMMY ${CMAKE_BUILD_TYPE})
FUNCTION(PREPEND var prefix)
SET(listVar "")
FOREACH(f ${ARGN})
LIST(APPEND listVar "${prefix}/${f}")
ENDFOREACH(f)
SET(${var} "${listVar}" PARENT_SCOPE)
ENDFUNCTION(PREPEND)
set(bindings_src ${PROJECT_SOURCE_DIR}/dom/bindings/codegen)
set(webidls_src ${PROJECT_SOURCE_DIR}/dom/webidls)
# Without Bindings/* stuff, since we install that separately below
set(globalgen_base_src
PrototypeList.rs
RegisterBindings.rs
InterfaceObjectMap.rs
InterfaceTypes.rs
InheritTypes.rs
UnionTypes.rs
)
set(globalgen_src
${globalgen_base_src}
Bindings/mod.rs
)
file(GLOB_RECURSE webidls ${webidls_src}/*.webidl)
string(REGEX REPLACE ";" "\n" webidl_filelist "${webidls}")
file(WRITE "${PROJECT_BINARY_DIR}/webidls.list" "${webidl_filelist}")
string(REGEX REPLACE "\\.webidl(;|$)" "\\1" bindings "${webidls}")
string(REGEX REPLACE "(^|;)${webidls_src}/" "\\1" bindings "${bindings}")
set(globalgen_deps
${bindings_src}/GlobalGen.py
${bindings_src}/Bindings.conf
${bindings_src}/Configuration.py
${bindings_src}/CodegenRust.py
${bindings_src}/parser/WebIDL.py
)
set(bindinggen_deps
${globalgen_deps}
${bindings_src}/BindingGen.py
)
add_custom_command(
OUTPUT Bindings
COMMAND ${CMAKE_COMMAND} -E make_directory Bindings
)
add_custom_command(
OUTPUT _cache
COMMAND ${CMAKE_COMMAND} -E make_directory _cache
)
# Specify python 2 as required
find_package( PythonInterp 2 REQUIRED )
add_custom_command(
OUTPUT ParserResults.pkl
COMMAND ${PYTHON_EXECUTABLE} -B ${bindings_src}/pythonpath.py -I ${bindings_src}/parser -I ${bindings_src}/ply
${bindings_src}/GlobalGen.py
--cachedir=_cache
--filelist=webidls.list
${bindings_src}/Bindings.conf
.
${PROJECT_SOURCE_DIR}
${PROJECT_BINARY_DIR}/../css-properties.json
${PROJECT_SOURCE_DIR}/../../target/doc/servo
DEPENDS Bindings _cache ${globalgen_deps} ${webidls} ${PROJECT_BINARY_DIR}/../css-properties.json
VERBATIM
)
# We need an intermediate custom target for this, due to this misfeature:
# > If any dependency is an OUTPUT of another custom command in the same
# > directory CMake automatically brings the other custom command into the
# > target in which this command is built.
# So, depending directly on ParserResults.pkl from the add_custom_command
# below would cause GlobalGen.py to be executed each time.
add_custom_target(ParserResults ALL DEPENDS ParserResults.pkl)
add_custom_target(generate-bindings ALL)
foreach(binding IN LISTS bindings)
add_custom_command(
OUTPUT Bindings/${binding}Binding.rs
COMMAND ${PYTHON_EXECUTABLE} -B ${bindings_src}/pythonpath.py -I ${bindings_src}/parser -I ${bindings_src}/ply
${bindings_src}/BindingGen.py
${bindings_src}/Bindings.conf
.
Bindings/${binding}Binding
${webidls_src}/${binding}.webidl
DEPENDS Bindings ${bindinggen_deps} ${webidls} ParserResults
VERBATIM
)
add_custom_target(${binding} DEPENDS Bindings/${binding}Binding.rs)
add_dependencies(generate-bindings ${binding})
endforeach()
PREPEND(globalgen_out ${CMAKE_BINARY_DIR}/ ${globalgen_base_src})
install(FILES ${globalgen_out} DESTINATION .)
install(DIRECTORY ${CMAKE_BINARY_DIR}/Bindings/ DESTINATION Bindings)

View file

@ -9,7 +9,7 @@ use std::fmt;
use std::fs::File; use std::fs::File;
use std::io::Write; use std::io::Write;
use std::path::PathBuf; use std::path::PathBuf;
use std::str; use std::process::Command;
use std::time::Instant; use std::time::Instant;
fn main() { fn main() {
@ -17,30 +17,22 @@ fn main() {
let style_out_dir = PathBuf::from(env::var_os("DEP_SERVO_STYLE_CRATE_OUT_DIR").unwrap()); let style_out_dir = PathBuf::from(env::var_os("DEP_SERVO_STYLE_CRATE_OUT_DIR").unwrap());
let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()); let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap());
let json = "css-properties.json"; let target_dir = PathBuf::from(env::var_os("CARGO_TARGET_DIR").unwrap());
std::fs::copy(style_out_dir.join(json), out_dir.join(json)).unwrap();
// This must use the Ninja generator -- it's the only one that let status = Command::new(find_python())
// parallelizes cmake's output properly. (Cmake generates .arg("dom/bindings/codegen/run.py")
// separate makefiles, each of which try to build .arg(style_out_dir.join("css-properties.json"))
// ParserResults.pkl, and then stomp on eachother.) .arg(&out_dir)
let mut build = cmake::Config::new("."); .arg(target_dir.join("doc").join("servo"))
.status()
let target = env::var("TARGET").unwrap(); .unwrap();
if target.contains("windows-msvc") { if !status.success() {
// We must use Ninja on Windows for this -- msbuild is painfully slow, std::process::exit(1)
// and ninja is easier to install than make.
build.generator("Ninja");
} }
build.build(); println!("Binding generation completed in {:?}", start.elapsed());
println!( let json = out_dir.join("InterfaceObjectMapData.json");
"Binding generation completed in {}s",
start.elapsed().as_secs()
);
let json = out_dir.join("build").join("InterfaceObjectMapData.json");
let json: Value = serde_json::from_reader(File::open(&json).unwrap()).unwrap(); let json: Value = serde_json::from_reader(File::open(&json).unwrap()).unwrap();
let mut map = phf_codegen::Map::new(); let mut map = phf_codegen::Map::new();
for (key, value) in json.as_object().unwrap() { for (key, value) in json.as_object().unwrap() {
@ -74,3 +66,27 @@ impl<'a> phf_shared::PhfHash for Bytes<'a> {
self.0.as_bytes().phf_hash(hasher) self.0.as_bytes().phf_hash(hasher)
} }
} }
fn find_python() -> String {
env::var("PYTHON").ok().unwrap_or_else(|| {
let candidates = if cfg!(windows) {
["python2.7.exe", "python27.exe", "python.exe"]
} else {
["python2.7", "python2", "python"]
};
for &name in &candidates {
if Command::new(name)
.arg("--version")
.output()
.ok()
.map_or(false, |out| out.status.success())
{
return name.to_owned();
}
}
panic!(
"Can't find python (tried {})! Try fixing PATH or setting the PYTHON env var",
candidates.join(", ")
)
})
}

View file

@ -1,54 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
import sys
import os
sys.path.append(os.path.join(".", "parser"))
sys.path.append(os.path.join(".", "ply"))
import cPickle
from Configuration import Configuration
from CodegenRust import CGBindingRoot, replaceFileIfChanged
def generate_binding_rs(config, outputprefix, webidlfile):
"""
|config| Is the configuration object.
|outputprefix| is a prefix to use for the header guards and filename.
"""
filename = outputprefix + ".rs"
module = CGBindingRoot(config, outputprefix, webidlfile).define()
if not module:
print "Skipping empty module: %s" % (filename)
elif replaceFileIfChanged(filename, module):
print "Generating binding implementation: %s" % (filename)
def main():
# Parse arguments.
from optparse import OptionParser
usagestring = "usage: %prog configFile outputdir outputPrefix webIDLFile"
o = OptionParser(usage=usagestring)
(options, args) = o.parse_args()
if len(args) != 4:
o.error(usagestring)
configFile = os.path.normpath(args[0])
outputdir = args[1]
outputPrefix = args[2]
webIDLFile = os.path.normpath(args[3])
# Load the parsing results
resultsPath = os.path.join(outputdir, 'ParserResults.pkl')
with open(resultsPath, 'rb') as f:
parserData = cPickle.load(f)
# Create the configuration data.
config = Configuration(configFile, parserData)
# Generate the prototype classes.
generate_binding_rs(config, outputPrefix, webIDLFile)
if __name__ == '__main__':
main()

View file

@ -54,32 +54,6 @@ RUST_KEYWORDS = {"abstract", "alignof", "as", "become", "box", "break", "const",
"use", "virtual", "where", "while", "yield"} "use", "virtual", "where", "while", "yield"}
def replaceFileIfChanged(filename, newContents):
"""
Read a copy of the old file, so that we don't touch it if it hasn't changed.
Returns True if the file was updated, false otherwise.
"""
# XXXjdm This doesn't play well with make right now.
# Force the file to always be updated, or else changing CodegenRust.py
# will cause many autogenerated bindings to be regenerated perpetually
# until the result is actually different.
# oldFileContents = ""
# try:
# with open(filename, 'rb') as oldFile:
# oldFileContents = ''.join(oldFile.readlines())
# except:
# pass
# if newContents == oldFileContents:
# return False
with open(filename, 'wb') as f:
f.write(newContents)
return True
def toStringBool(arg): def toStringBool(arg):
return str(not not arg).lower() return str(not not arg).lower()

View file

@ -1,143 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
# We do one global pass over all the WebIDL to generate our prototype enum
# and generate information for subsequent phases.
import sys
import os
import json
sys.path.append(os.path.join(".", "parser"))
sys.path.append(os.path.join(".", "ply"))
import WebIDL
import cPickle
from Configuration import Configuration
from CodegenRust import GlobalGenRoots, replaceFileIfChanged
def generate_file(config, name, filename):
root = getattr(GlobalGenRoots, name)(config)
code = root.define()
if replaceFileIfChanged(filename, code):
print "Generating %s" % (filename)
else:
print "%s hasn't changed - not touching it" % (filename)
def main():
# Parse arguments.
from optparse import OptionParser
usageString = "usage: %prog [options] configFile outputdir webidldir cssProperties.json docServoDir [files]"
o = OptionParser(usage=usageString)
o.add_option("--cachedir", dest='cachedir', default=None,
help="Directory in which to cache lex/parse tables.")
o.add_option("--filelist", dest='filelist', default=None,
help="A file containing the list (one per line) of webidl files to process.")
(options, args) = o.parse_args()
if len(args) < 2:
o.error(usageString)
configFile = args[0]
outputdir = args[1]
baseDir = args[2]
css_properties_json = args[3]
doc_servo = args[4]
if options.filelist is not None:
fileList = [l.strip() for l in open(options.filelist).xreadlines()]
else:
fileList = args[3:]
# Parse the WebIDL.
parser = WebIDL.Parser(options.cachedir)
for filename in fileList:
fullPath = os.path.normpath(os.path.join(baseDir, filename))
with open(fullPath, 'rb') as f:
lines = f.readlines()
parser.parse(''.join(lines), fullPath)
add_css_properties_attributes(fileList, css_properties_json, parser)
parserResults = parser.finish()
# Write the parser results out to a pickle.
resultsPath = os.path.join(outputdir, 'ParserResults.pkl')
with open(resultsPath, 'wb') as resultsFile:
cPickle.dump(parserResults, resultsFile, -1)
# Load the configuration.
config = Configuration(configFile, parserResults)
to_generate = [
('PrototypeList', 'PrototypeList.rs'),
('RegisterBindings', 'RegisterBindings.rs'),
('InterfaceObjectMap', 'InterfaceObjectMap.rs'),
('InterfaceObjectMapData', 'InterfaceObjectMapData.json'),
('InterfaceTypes', 'InterfaceTypes.rs'),
('InheritTypes', 'InheritTypes.rs'),
('Bindings', os.path.join('Bindings', 'mod.rs')),
('UnionTypes', 'UnionTypes.rs'),
]
for name, filename in to_generate:
generate_file(config, name, os.path.join(outputdir, filename))
generate_file(config, 'SupportedDomApis', os.path.join(doc_servo, 'apis.html'))
def add_css_properties_attributes(webidl_files, css_properties_json, parser):
for filename in webidl_files:
if os.path.basename(filename) == "CSSStyleDeclaration.webidl":
break
else:
return
css_properties = json.load(open(css_properties_json, "rb"))
idl = "partial interface CSSStyleDeclaration {\n%s\n};\n" % "\n".join(
" [%sCEReactions, SetterThrows] attribute [TreatNullAs=EmptyString] DOMString %s;" % (
('Pref="%s", ' % data["pref"] if data["pref"] else ""),
attribute_name
)
for (kind, properties_list) in sorted(css_properties.items())
for (property_name, data) in sorted(properties_list.items())
for attribute_name in attribute_names(property_name)
)
parser.parse(idl.encode("utf-8"), "CSSStyleDeclaration_generated.webidl")
def attribute_names(property_name):
# https://drafts.csswg.org/cssom/#dom-cssstyledeclaration-dashed-attribute
if property_name != "float":
yield property_name
else:
yield "_float"
# https://drafts.csswg.org/cssom/#dom-cssstyledeclaration-camel-cased-attribute
if "-" in property_name:
yield "".join(camel_case(property_name))
# https://drafts.csswg.org/cssom/#dom-cssstyledeclaration-webkit-cased-attribute
if property_name.startswith("-webkit-"):
yield "".join(camel_case(property_name), True)
# https://drafts.csswg.org/cssom/#css-property-to-idl-attribute
def camel_case(chars, webkit_prefixed=False):
if webkit_prefixed:
chars = chars[1:]
next_is_uppercase = False
for c in chars:
if c == '-':
next_is_uppercase = True
elif next_is_uppercase:
next_is_uppercase = False
# Should be ASCII-uppercase, but all non-custom CSS property names are within ASCII
yield c.upper()
else:
yield c
if __name__ == '__main__':
main()

View file

@ -1,61 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
"""
Run a python script, adding extra directories to the python path.
"""
def main(args):
def usage():
print >>sys.stderr, "pythonpath.py -I directory script.py [args...]"
sys.exit(150)
paths = []
while True:
try:
arg = args[0]
except IndexError:
usage()
if arg == '-I':
args.pop(0)
try:
path = args.pop(0)
except IndexError:
usage()
paths.append(os.path.abspath(path))
continue
if arg.startswith('-I'):
paths.append(os.path.abspath(args.pop(0)[2:]))
continue
if arg.startswith('-D'):
os.chdir(args.pop(0)[2:])
continue
break
script = args[0]
sys.path[0:0] = [os.path.abspath(os.path.dirname(script))] + paths
sys.argv = args
sys.argc = len(args)
frozenglobals['__name__'] = '__main__'
frozenglobals['__file__'] = script
execfile(script, frozenglobals)
# Freeze scope here ... why this makes things work I have no idea ...
frozenglobals = globals()
import sys
import os
if __name__ == '__main__':
main(sys.argv[1:])

View file

@ -0,0 +1,119 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
import os
import sys
import json
def main():
os.chdir(os.path.join(os.path.dirname(__file__)))
sys.path[0:0] = ["./parser", "./ply"]
css_properties_json, out_dir, doc_servo = sys.argv[1:]
webidls_dir = "../../webidls"
config_file = "Bindings.conf"
import WebIDL
from Configuration import Configuration
from CodegenRust import CGBindingRoot
parser = WebIDL.Parser(make_dir(out_dir, "cache"))
webidls = [name for name in os.listdir(webidls_dir) if name.endswith(".webidl")]
for webidl in webidls:
filename = os.path.join(webidls_dir, webidl)
with open(filename, "rb") as f:
parser.parse(f.read(), filename)
add_css_properties_attributes(css_properties_json, parser)
parser_results = parser.finish()
config = Configuration(config_file, parser_results)
make_dir(out_dir, "Bindings")
for name, filename in [
("PrototypeList", "PrototypeList.rs"),
("RegisterBindings", "RegisterBindings.rs"),
("InterfaceObjectMap", "InterfaceObjectMap.rs"),
("InterfaceObjectMapData", "InterfaceObjectMapData.json"),
("InterfaceTypes", "InterfaceTypes.rs"),
("InheritTypes", "InheritTypes.rs"),
("Bindings", "Bindings/mod.rs"),
("UnionTypes", "UnionTypes.rs"),
]:
generate(config, name, os.path.join(out_dir, filename))
make_dir(doc_servo)
generate(config, "SupportedDomApis", os.path.join(doc_servo, "apis.html"))
for webidl in webidls:
filename = os.path.join(webidls_dir, webidl)
prefix = "Bindings/%sBinding" % webidl[:-len(".webidl")]
module = CGBindingRoot(config, prefix, filename).define()
if module:
with open(os.path.join(out_dir, prefix + ".rs"), "wb") as f:
f.write(module)
def make_dir(out_dir, nested=""):
path = os.path.join(out_dir, nested)
if not os.path.exists(path):
os.makedirs(path)
return path
def generate(config, name, filename):
from CodegenRust import GlobalGenRoots
root = getattr(GlobalGenRoots, name)(config)
code = root.define()
with open(filename, "wb") as f:
f.write(code)
def add_css_properties_attributes(css_properties_json, parser):
css_properties = json.load(open(css_properties_json, "rb"))
idl = "partial interface CSSStyleDeclaration {\n%s\n};\n" % "\n".join(
" [%sCEReactions, SetterThrows] attribute [TreatNullAs=EmptyString] DOMString %s;" % (
('Pref="%s", ' % data["pref"] if data["pref"] else ""),
attribute_name
)
for (kind, properties_list) in sorted(css_properties.items())
for (property_name, data) in sorted(properties_list.items())
for attribute_name in attribute_names(property_name)
)
parser.parse(idl.encode("utf-8"), "CSSStyleDeclaration_generated.webidl")
def attribute_names(property_name):
# https://drafts.csswg.org/cssom/#dom-cssstyledeclaration-dashed-attribute
if property_name != "float":
yield property_name
else:
yield "_float"
# https://drafts.csswg.org/cssom/#dom-cssstyledeclaration-camel-cased-attribute
if "-" in property_name:
yield "".join(camel_case(property_name))
# https://drafts.csswg.org/cssom/#dom-cssstyledeclaration-webkit-cased-attribute
if property_name.startswith("-webkit-"):
yield "".join(camel_case(property_name), True)
# https://drafts.csswg.org/cssom/#css-property-to-idl-attribute
def camel_case(chars, webkit_prefixed=False):
if webkit_prefixed:
chars = chars[1:]
next_is_uppercase = False
for c in chars:
if c == '-':
next_is_uppercase = True
elif next_is_uppercase:
next_is_uppercase = False
# Should be ASCII-uppercase, but all non-custom CSS property names are within ASCII
yield c.upper()
else:
yield c
if __name__ == "__main__":
main()

View file

@ -206,10 +206,7 @@
pub mod macros; pub mod macros;
pub mod types { pub mod types {
#[cfg(not(target_env = "msvc"))]
include!(concat!(env!("OUT_DIR"), "/InterfaceTypes.rs")); include!(concat!(env!("OUT_DIR"), "/InterfaceTypes.rs"));
#[cfg(target_env = "msvc")]
include!(concat!(env!("OUT_DIR"), "/build/InterfaceTypes.rs"));
} }
pub mod abstractworker; pub mod abstractworker;

View file

@ -27,52 +27,28 @@ mod build_gecko {
pub fn generate() {} pub fn generate() {}
} }
#[cfg(windows)]
fn find_python() -> String {
if Command::new("python2.7.exe")
.arg("--version")
.output()
.is_ok()
{
return "python2.7.exe".to_owned();
}
if Command::new("python27.exe")
.arg("--version")
.output()
.is_ok()
{
return "python27.exe".to_owned();
}
if Command::new("python.exe").arg("--version").output().is_ok() {
return "python.exe".to_owned();
}
panic!(concat!(
"Can't find python (tried python2.7.exe, python27.exe, and python.exe)! ",
"Try fixing PATH or setting the PYTHON env var"
));
}
#[cfg(not(windows))]
fn find_python() -> String {
if Command::new("python2.7")
.arg("--version")
.output()
.unwrap()
.status
.success()
{
"python2.7"
} else {
"python"
}
.to_owned()
}
lazy_static! { lazy_static! {
pub static ref PYTHON: String = env::var("PYTHON").ok().unwrap_or_else(find_python); pub static ref PYTHON: String = env::var("PYTHON").ok().unwrap_or_else(|| {
let candidates = if cfg!(windows) {
["python2.7.exe", "python27.exe", "python.exe"]
} else {
["python2.7", "python2", "python"]
};
for &name in &candidates {
if Command::new(name)
.arg("--version")
.output()
.ok()
.map_or(false, |out| out.status.success())
{
return name.to_owned();
}
}
panic!(
"Can't find python (tried {})! Try fixing PATH or setting the PYTHON env var",
candidates.join(", ")
)
});
} }
fn generate_properties(engine: &str) { fn generate_properties(engine: &str) {