Merge branch 'master' into mh-fix-379

Author: mhucka

.github/workflows/pythonpackage.yml

@@ -38,6 +38,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
+          pip install -r requirements.txt
           python dev_tools/write-ci-requirements.py --relative-cirq-version=${{ matrix.cirq-version }} --all-extras
           pip install -r ci-requirements.txt
           pip install --no-deps -e .
@@ -52,7 +53,6 @@ jobs:
 
       - name: Test with pytest
         run: |
-          pip install pytest
           # RECIRQ_IMPORT_FAILSAFE: skip tests on unsupported Cirq configurations
           # EXPORT_OMP_NUM_THREADS: pyscf has poor openmp performance which slows down qcqmc tests.
           export OMP_NUM_THREADS=1

CONTRIBUTING.md

@@ -69,7 +69,49 @@ Guides](https://google.github.io/styleguide/) in your code, particularly the
 
 ### Development and testing
 
-Do your work and `git commit` your changes to your branch as needed.
+Before you begin developing ReCirq experiments and modules, we recommend you
+first create a virtual Python environment. You can use either Python's built-in
+[`venv`](https://docs.python.org/3/library/venv.html) module or another tool
+that you are comfortable with.
+
+Then, in that virtual environment, install the ReCirq dependencies using `pip`:
+
+```shell
+pip install -r requirements.txt
+```
+
+You _may_ need to install additional requirements, depending on the ReCirq
+experiment you want to work on. These additional requirements will be in a
+file named `extra-requirements.txt` in the experiment's subdirectory under
+`recirq/`. For example, if you were working with `recirq/optimize`, you would
+need to install its extra dependencies like this:
+
+```shell
+pip install -r recirq/optimize/extra-requirements.txt
+```
+
+Finally, if you are going to edit any of the Jupyter notebooks, install the
+additional requirements needed to run the notebook format checks:
+
+```shell
+pip install -r dev_tools/requirements/deps/tensorflow-docs.txt
+```
+
+Once the environment is set up, you can do your work and `git commit` your
+changes to your branch as needed.
+
+To test notebooks for proper formatting and other issues, run the following
+command:
+
+```shell
+dev_tools/nbformat
+```
+
+To test your code, run
+
+```shell
+pytest recirq
+```
 
 ### Pull requests and code reviews
 

recirq/documentation_utils.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import inspect
+import io
 import os
 import re
 import tarfile
@@ -67,6 +68,14 @@ def fetch_guide_data_collection_data(base_dir=None):
 
     print("Downloading guide/data_collection data.")
     stream = urllib.request.urlopen("https://ndownloader.figshare.com/files/25541666")
-
-    with tarfile.open(fileobj=stream, mode='r|xz') as tf:
-        tf.extractall(path=base_dir)
+    # Read into a BytesIO object to make it seekable
+    stream_bytes = io.BytesIO(stream.read())
+
+    with tarfile.open(fileobj=stream_bytes, mode='r|xz') as tf:
+        for member in tf.getmembers():
+            # Ensure the path being extracted is safe.
+            if not os.path.abspath(os.path.join(base_dir, member.name)).startswith(
+                os.path.abspath(base_dir)
+            ):
+                raise ValueError(f"Encountered untrusted path {member.name}")
+            tf.extract(member, path=base_dir)

recirq/documentation_utils_test.py

@@ -12,6 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import os
+import io
+import tarfile
+from unittest import mock
+
+import pytest
 
 import recirq
 from recirq.readout_scan.tasks import ReadoutScanTask
@@ -35,6 +40,23 @@ def test_display_markdown_docstring():
 """
 
 
-def test_fetch_guide_data_collection_data(tmpdir):
-    recirq.fetch_guide_data_collection_data(base_dir=tmpdir)
-    assert os.path.exists(f'{tmpdir}/2020-02-tutorial')
+@mock.patch('urllib.request.urlopen')
+def test_fetch_guide_data_collection_data_traversal(mock_urlopen, tmpdir):
+    # Create a malicious tarball in memory.
+    malicious_tar_stream = io.BytesIO()
+    with tarfile.open(fileobj=malicious_tar_stream, mode='w:xz') as tf:
+        # Add a file that tries to write outside the target directory
+        malicious_info = tarfile.TarInfo(name="../../tmp/pwned")
+        tf.addfile(malicious_info, io.BytesIO(b"pwned"))
+    malicious_tar_stream.seek(0)
+
+    # Read the stream into a BytesIO object so that the mock should return a
+    # response object whose read() method returns the tarball content.
+    mock_response = mock.Mock()
+    mock_response.read.return_value = malicious_tar_stream.getvalue()
+    mock_urlopen.return_value = mock_response
+
+    with pytest.raises(ValueError, match="Encountered untrusted path"):
+        recirq.fetch_guide_data_collection_data(base_dir=tmpdir)
+
+    assert not os.path.exists('/tmp/pwned')

requirements.txt

@@ -6,3 +6,4 @@ seaborn
 sphinx
 ipython
 black
+pytest