Auto merge of #19244 - asajeffrey:test-perf-submit-to-s3, r=jdm

Submit test-perf CSV files to S3

<!-- Please describe your changes on the following line: -->

Submit CSV files to S3 rather than json files to Perfherder.

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: -->
- [X] `./mach build -d` does not report any errors
- [X] `./mach test-tidy` does not report any errors
- [X] These changes do not require tests because this is test infrastructure

<!-- Also, please make sure that "Allow edits from maintainers" checkbox is checked, so that we can help you if you get stuck somewhere along the way.-->

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/19244)
<!-- Reviewable:end -->

etc/ci/performance/set_s3_policy.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import boto3
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=("Set the policy of the servo-perf bucket. "
+                     "Remember to set your S3 credentials "
+                     "https://github.com/boto/boto3"))
+    parser.parse_args()
+
+    s3 = boto3.resource('s3')
+    BUCKET = 'servo-perf'
+    POLICY = """{
+  "Version":"2012-10-17",
+  "Statement":[
+    {
+      "Effect":"Allow",
+      "Principal":"*",
+      "Action":[
+        "s3:ListBucket",
+        "s3:GetBucketLocation"
+      ],
+      "Resource":"arn:aws:s3:::servo-perf"
+    },
+    {
+      "Effect":"Allow",
+      "Principal":"*",
+      "Action":[
+        "s3:GetObject",
+        "s3:GetObjectAcl"
+      ],
+      "Resource":"arn:aws:s3:::servo-perf/*"
+    }
+  ]
+}"""
+
+    s3.BucketPolicy(BUCKET).put(Policy=POLICY)
+
+    print("Done!")
+
+
+if __name__ == "__main__":
+    main()

etc/ci/performance/submit_to_s3.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import boto3
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=("Submit Servo performance data to S3. "
+                     "Remember to set your S3 credentials "
+                     "https://github.com/boto/boto3"))
+    parser.add_argument("perf_file",
+                        help="the output CSV file from runner")
+    parser.add_argument("perf_key",
+                        help="the S3 key to upload to")
+    args = parser.parse_args()
+
+    s3 = boto3.client('s3')
+    BUCKET = 'servo-perf'
+    s3.upload_file(args.perf_file, BUCKET, args.perf_key)
+
+    print("Done!")
+
+
+if __name__ == "__main__":
+    main()

etc/ci/performance/test_all.sh

@@ -48,7 +48,8 @@ python3 -m http.server > /dev/null 2>&1 &
 # MANIFEST="page_load_test/tp5n/20160509.manifest"
 MANIFEST="page_load_test/test.manifest" # A manifest that excludes
                                         # timeout test cases
-PERF_FILE="output/perf-$(uname -s)-$(uname -m)-$(date +%s).csv"
+PERF_KEY="perf-$(uname -s)-$(uname -m)-$(date +%s).csv"
+PERF_FILE="output/${PERF_KEY}"
 
 echo "Running tests"
 python3 runner.py ${engine} --runs 4 --timeout "${timeout}" --base "${base}" \
@@ -56,14 +57,8 @@ python3 runner.py ${engine} --runs 4 --timeout "${timeout}" --base "${base}" \
 
 if [[ "${submit:-}" ]];
 then
-    echo "Submitting to Perfherder"
-    # Perfherder SSL check will fail if time is not accurate,
-    # sync time before you submit
-    # TODO: we are using Servo's revision hash for Gecko's result to make both
-    # results appear on the same date. Use the correct result when Perfherder
-    # allows us to change the date.
-    python3 submit_to_perfherder.py \
-            "${engine}" "${PERF_FILE}" servo/revision.json
+    echo "Submitting to S3"
+    python3 submit_to_s3.py "${PERF_FILE}" "${PERF_KEY}"
 fi
 
 echo "Stopping the local server"

etc/ci/performance/test_perf.sh

@@ -29,7 +29,7 @@ fi
 virtualenv venv --python="$(which python3)"
 PS1="" source venv/bin/activate
 # `PS1` must be defined before activating virtualenv
-pip install "treeherder-client>=3.0.0"
+pip install "boto3>=1.4.0"
 
 mkdir -p servo
 mkdir -p output # Test result will be saved to output/perf-<timestamp>.json

python/servo/testing_commands.py

@@ -186,12 +186,6 @@ class MachCommands(CommandBase):
         if base:
             cmd += ["--base", base]
         if submit:
-            if not ("TREEHERDER_CLIENT_ID" in os.environ and
-                    "TREEHERDER_CLIENT_SECRET" in os.environ):
-                print("Please set the environment variable \"TREEHERDER_CLIENT_ID\""
-                      " and \"TREEHERDER_CLIENT_SECRET\" to submit the performance"
-                      " test result to perfherder")
-                return 1
             cmd += ["--submit"]
         return call(cmd,
                     env=env,