Auto merge of #12186 - GuillaumeGomez:video-metadata, r=larsbergstrom,jdm,KiChjang

Implement video-metadata check

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="35" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/12186)
<!-- Reviewable:end -->

.travis.yml

@@ -36,6 +36,9 @@ matrix:
             - xorg-dev
             - ccache
             - libdbus-glib-1-dev
+            - libavformat-dev
+            - libavcodec-dev
+            - libavutil-dev
 
 branches:
   only:

README.md

@@ -43,7 +43,8 @@ sudo apt-get install git curl freeglut3-dev autoconf \
     libfreetype6-dev libgl1-mesa-dri libglib2.0-dev xorg-dev \
     gperf g++ build-essential cmake virtualenv python-pip \
     libssl-dev libbz2-dev libosmesa6-dev libxmu6 libxmu-dev \
-    libglu1-mesa-dev libgles2-mesa-dev libegl1-mesa-dev libdbus-1-dev
+    libglu1-mesa-dev libgles2-mesa-dev libegl1-mesa-dev libdbus-1-dev \
+    libavformat-dev
 ```
 If you are on **Ubuntu 14.04** and encountered errors on installing these dependencies involving `libcheese`, see [#6158](https://github.com/servo/servo/issues/6158) for a workaround.
 
@@ -56,7 +57,7 @@ sudo dnf install curl freeglut-devel libtool gcc-c++ libXi-devel \
     freetype-devel mesa-libGL-devel mesa-libEGL-devel glib2-devel libX11-devel libXrandr-devel gperf \
     fontconfig-devel cabextract ttmkfdir python python-virtualenv python-pip expat-devel \
     rpm-build openssl-devel cmake bzip2-devel libXcursor-devel libXmu-devel mesa-libOSMesa-devel \
-    dbus-devel
+    dbus-devel ffmpeg-devel
 ```
 
 On Arch Linux:
@@ -89,7 +90,7 @@ pacman -Su
 pacman -Sy git mingw-w64-x86_64-toolchain mingw-w64-x86_64-freetype \
     mingw-w64-x86_64-icu mingw-w64-x86_64-nspr mingw-w64-x86_64-ca-certificates \
     mingw-w64-x86_64-expat mingw-w64-x86_64-cmake tar diffutils patch \
-    patchutils make python2-setuptools
+    patchutils make python2-setuptools mingw-w64-x86_64-ffmpeg
 export GCC_URL=http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-gcc
 export GCC_EXT=5.4.0-1-any.pkg.tar.xz
 pacman -U --noconfirm $GCC_URL-$GCC_EXT $GCC_URL-ada-$GCC_EXT \

appveyor.yml

@@ -19,7 +19,7 @@ cache:
 install:
   - bash -lc "echo $MSYSTEM; pacman --needed --noconfirm -Sy pacman-mirrors"
   - bash -lc "pacman --noconfirm -Sy"
-  - bash -lc "pacman -Sy --needed --noconfirm git mingw-w64-x86_64-toolchain mingw-w64-x86_64-freetype mingw-w64-x86_64-icu mingw-w64-x86_64-nspr mingw-w64-x86_64-ca-certificates mingw-w64-x86_64-expat mingw-w64-x86_64-cmake tar diffutils patch patchutils make python2-setuptools"
+  - bash -lc "pacman -Sy --needed --noconfirm git mingw-w64-x86_64-toolchain mingw-w64-x86_64-freetype mingw-w64-x86_64-icu mingw-w64-x86_64-nspr mingw-w64-x86_64-ca-certificates mingw-w64-x86_64-expat mingw-w64-x86_64-cmake tar diffutils patch patchutils make python2-setuptools mingw-w64-x86_64-ffmpeg"
   - bash -lc "easy_install-2.7 pip virtualenv"
   - bash -lc "mv /mingw64/bin/python2.exe /mingw64/bin/python2-mingw64.exe"
   - bash -lc "mv /mingw64/bin/python2.7.exe /mingw64/bin/python2.7-mingw64.exe"

components/script/Cargo.toml

@@ -17,6 +17,9 @@ debugmozjs = ['js/debugmozjs']
 [target.'cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))'.dependencies]
 tinyfiledialogs = {git = "https://github.com/jdm/tinyfiledialogs"}
 
+[target.'cfg(not(any(target_os = "android", target_arch = "arm", target_arch = "aarch64")))'.dependencies]
+video-metadata = {git = "https://github.com/GuillaumeGomez/video-metadata-rs"}
+
 [dependencies]
 angle = {git = "https://github.com/servo/angle", branch = "servo"}
 app_units = "0.2.5"

components/script/dom/bindings/trace.rs

@@ -91,6 +91,7 @@ use style::element_state::*;
 use style::properties::PropertyDeclarationBlock;
 use style::selector_impl::{PseudoElement, ElementSnapshot};
 use style::values::specified::Length;
+use time::Duration;
 use url::Origin as UrlOrigin;
 use url::Url;
 use uuid::Uuid;
@@ -109,6 +110,8 @@ no_jsmanaged_fields!(EncodingRef);
 
 no_jsmanaged_fields!(Reflector);
 
+no_jsmanaged_fields!(Duration);
+
 /// Trace a `JSVal`.
 pub fn trace_jsval(tracer: *mut JSTracer, description: &str, val: &Heap<JSVal>) {
     unsafe {

components/script/dom/htmlmediaelement.rs

@@ -35,6 +35,8 @@ use string_cache::Atom;
 use task_source::TaskSource;
 use time::{self, Timespec, Duration};
 use url::Url;
+#[cfg(not(any(target_os = "android", target_arch = "arm", target_arch = "aarch64")))]
+use video_metadata;
 
 struct HTMLMediaElementContext {
     /// The element that initiated the request.
@@ -75,12 +77,11 @@ impl AsyncResponseListener for HTMLMediaElementContext {
         }
     }
 
-    fn data_available(&mut self, payload: Vec<u8>) {
+    fn data_available(&mut self, mut payload: Vec<u8>) {
         if self.ignore_response {
             return;
         }
 
-        let mut payload = payload;
         self.data.append(&mut payload);
 
         let elem = self.elem.root();
@@ -88,11 +89,7 @@ impl AsyncResponseListener for HTMLMediaElementContext {
         // https://html.spec.whatwg.org/multipage/#media-data-processing-steps-list
         // => "Once enough of the media data has been fetched to determine the duration..."
         if !self.have_metadata {
-            //TODO: actually check if the payload contains the full metadata
+            self.check_metadata(&elem);
-
-            // Step 6
-            elem.change_ready_state(HAVE_METADATA);
-            self.have_metadata = true;
         } else {
             elem.change_ready_state(HAVE_CURRENT_DATA);
         }
@@ -162,6 +159,46 @@ impl HTMLMediaElementContext {
             ignore_response: false,
         }
     }
+
+    #[cfg(not(any(target_os = "android", target_arch = "arm", target_arch = "aarch64")))]
+    fn check_metadata(&mut self, elem: &HTMLMediaElement) {
+        match video_metadata::get_format_from_slice(&self.data) {
+            Ok(meta) => {
+                let dur = meta.duration.unwrap_or(::std::time::Duration::new(0, 0));
+                *elem.video.borrow_mut() = Some(VideoMedia {
+                    format: format!("{:?}", meta.format),
+                    duration: Duration::seconds(dur.as_secs() as i64) +
+                              Duration::nanoseconds(dur.subsec_nanos() as i64),
+                    width: meta.size.width,
+                    height: meta.size.height,
+                    video: meta.video,
+                    audio: meta.audio,
+                });
+                // Step 6
+                elem.change_ready_state(HAVE_METADATA);
+                self.have_metadata = true;
+            }
+            _ => {}
+        }
+    }
+
+    #[cfg(any(target_os = "android", target_arch = "arm", target_arch = "aarch64"))]
+    fn check_metadata(&mut self, elem: &HTMLMediaElement) {
+        // Step 6.
+        elem.change_ready_state(HAVE_METADATA);
+        self.have_metadata = true;
+    }
+}
+
+#[derive(JSTraceable, HeapSizeOf)]
+pub struct VideoMedia {
+    format: String,
+    #[ignore_heap_size_of = "defined in time"]
+    duration: Duration,
+    width: u32,
+    height: u32,
+    video: String,
+    audio: Option<String>,
 }
 
 #[dom_struct]
@@ -175,6 +212,7 @@ pub struct HTMLMediaElement {
     error: MutNullableHeap<JS<MediaError>>,
     paused: Cell<bool>,
     autoplaying: Cell<bool>,
+    video: DOMRefCell<Option<VideoMedia>>,
 }
 
 impl HTMLMediaElement {
@@ -192,6 +230,7 @@ impl HTMLMediaElement {
             error: Default::default(),
             paused: Cell::new(true),
             autoplaying: Cell::new(true),
+            video: DOMRefCell::new(None),
         }
     }
 

components/script/lib.rs

@@ -87,6 +87,8 @@ extern crate url;
 #[macro_use]
 extern crate util;
 extern crate uuid;
+#[cfg(not(any(target_os = "android", target_arch = "arm", target_arch = "aarch64")))]
+extern crate video_metadata;
 extern crate webrender_traits;
 extern crate websocket;
 extern crate xml5ever;

components/servo/Cargo.lock

@@ -1904,6 +1904,7 @@ dependencies = [
  "url 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "util 0.0.1",
  "uuid 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "video-metadata 0.1.2 (git+https://github.com/GuillaumeGomez/video-metadata-rs)",
  "webrender_traits 0.2.0 (git+https://github.com/servo/webrender_traits)",
  "websocket 0.17.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "xml5ever 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2442,6 +2443,16 @@ dependencies = [
  "serde 0.7.15 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "video-metadata"
+version = "0.1.2"
+source = "git+https://github.com/GuillaumeGomez/video-metadata-rs#bc7f45023765b0ff7a7235e02bac3bfe9f24acd7"
+dependencies = [
+ "gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "void"
 version = "1.0.2"

ports/cef/Cargo.lock

@@ -1758,6 +1758,7 @@ dependencies = [
  "url 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "util 0.0.1",
  "uuid 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "video-metadata 0.1.2 (git+https://github.com/GuillaumeGomez/video-metadata-rs)",
  "webrender_traits 0.2.0 (git+https://github.com/servo/webrender_traits)",
  "websocket 0.17.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "xml5ever 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2304,6 +2305,16 @@ dependencies = [
  "serde 0.7.15 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "video-metadata"
+version = "0.1.2"
+source = "git+https://github.com/GuillaumeGomez/video-metadata-rs#bc7f45023765b0ff7a7235e02bac3bfe9f24acd7"
+dependencies = [
+ "gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "void"
 version = "1.0.2"

tests/wpt/metadata/html/semantics/embedded-content/media-elements/error-codes/error.html.ini

@@ -0,0 +1,6 @@
+[error.html]
+  type: testharness
+  expected: TIMEOUT
+  [audio.error after successful load]
+    expected: TIMEOUT
+

tests/wpt/metadata/html/semantics/embedded-content/media-elements/event_loadeddata.html.ini

@@ -7,3 +7,6 @@
   [setting src attribute on autoplay video should trigger loadeddata event]
     expected: NOTRUN
 
+  [setting src attribute on autoplay audio should trigger loadeddata event]
+    expected: NOTRUN
+

tests/wpt/metadata/html/semantics/embedded-content/media-elements/event_loadeddata_noautoplay.html.ini

@@ -0,0 +1,6 @@
+[event_loadeddata_noautoplay.html]
+  type: testharness
+  expected: TIMEOUT
+  [setting src attribute on non-autoplay audio should trigger loadeddata event]
+    expected: NOTRUN
+

tests/wpt/metadata/html/semantics/embedded-content/media-elements/event_loadedmetadata.html.ini

@@ -0,0 +1,6 @@
+[event_loadedmetadata.html]
+  type: testharness
+  expected: TIMEOUT
+  [setting src attribute on autoplay audio should trigger loadedmetadata event]
+    expected: NOTRUN
+

tests/wpt/metadata/html/semantics/embedded-content/media-elements/event_loadedmetadata_noautoplay.html.ini

@@ -0,0 +1,6 @@
+[event_loadedmetadata_noautoplay.html]
+  type: testharness
+  expected: TIMEOUT
+  [setting src attribute on non-autoplay audio should trigger loadedmetadata event]
+    expected: NOTRUN
+

tests/wpt/metadata/html/semantics/embedded-content/media-elements/event_order_loadedmetadata_loadeddata.html.ini

@@ -0,0 +1,6 @@
+[event_order_loadedmetadata_loadeddata.html]
+  type: testharness
+  expected: TIMEOUT
+  [setting src attribute on autoplay audio should trigger loadedmetadata then loadeddata event]
+    expected: NOTRUN
+

tests/wpt/metadata/html/semantics/embedded-content/media-elements/readyState_during_loadeddata.html.ini

@@ -0,0 +1,6 @@
+[readyState_during_loadeddata.html]
+  type: testharness
+  expected: TIMEOUT
+  [audio.readyState should be >= HAVE_CURRENT_DATA during loadeddata event]
+    expected: NOTRUN
+

tests/wpt/metadata/html/semantics/embedded-content/media-elements/readyState_during_loadedmetadata.html.ini

@@ -0,0 +1,6 @@
+[readyState_during_loadedmetadata.html]
+  type: testharness
+  expected: TIMEOUT
+  [audio.readyState should be >= HAVE_METADATA during loadedmetadata event]
+    expected: NOTRUN
+