Merge patch of diff of master and 1.13.0 tag

There was a network hiccup during hubflow release finish,
and 1.13 code ended up only being merged to develop, not master.

Author: Charles Overbeck

.gitallowed

@@ -1,3 +1,4 @@
 #initial .gitallowed, in the future this is where git-secrets false-positives should be added
 
 scripts/tests/test-webservice-image-digest.py:.*sha256
+scripts/webservice-image-digest.py:.*sha256

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,9 @@
+**Description**
+A description of the PR, should include a decent explanation as to why this change was needed and a decent explanation as to what this change does
+
+**Issue**
+A link to a github issue or SEAB- ticket (using that as a prefix)
+
+Please make sure that you've checked the following before submitting your pull request. Thanks!
+
+- [ ] Ensure that the PR targets the correct branch. Check the milestone or fix version of the ticket.

SECURITY.md

@@ -0,0 +1,21 @@
+# Security Policy
+
+## Supported Versions
+
+The Dockstore system routinely receives security updates to the most recently
+released tagged minor version. All previous versions are unsupported.
+
+## Reporting a Vulnerability
+
+Users are able to open helpdesk tickets on [Discourse](https://discuss.dockstore.org/). Users can create helpdesk tickets in case of privacy complaints, security vulnerabilities, or any other urgent matter related to Dockstore. Helpdesk tickets will be addressed by Dockstore administrators.
+
+The following steps can be taken to create a helpdesk ticket (also shown [here](https://discuss.dockstore.org/t/opening-helpdesk-tickets/1506)).
+
+1. Navigate to [Discourse](https://discuss.dockstore.org/) and login.
+2. Select your profile icon, located in the top right corner of the screen.
+3. Select the `mail` icon, located in the dropdown.
+4. Send a message to the `dockstore_admins` group.
+
+Note
+
+> If you are unable to see a New Message button on the mail page, you may be considered a new user and have insufficient privileges. Entering 5 topics and viewing 30 posts over a minimum of 10 minutes will raise your privileges. You will be notified of any privilege changes to your account via the mailbox.

docker-compose.yml

@@ -24,7 +24,7 @@ services:
       - log_volume:/dockstore_logs
       - ./config/web.yml:/home/web.yml
       - ./config/init_webservice.sh:/home/init_webservice.sh
-      - ${GITHUB_APP_PRIVATE_KEY_FILE}:/home/dockstore_github_app_private_key.pem
+      - ${GITHUB_APP_PRIVATE_KEY_FILE}:/dockstore/github-key/dockstore-github-private-key.pem
       - ../language-plugins/:/root/.dockstore/language-plugins
     command: ["bash", "/home/init_webservice.sh"]
     ports:

dockstore_launcher_config/compose.config

@@ -1,37 +1,48 @@
 {
-"PUBLIC_LAUNCHER_IP_ADDRESS":"",
-"PRODUCTION":false,
-"DOCKSTORE_VERSION":"develop",
-"UI2_HASH":"develop",
-"GITHUB_CLIENT2_ID":"foobared",
-"GITHUB_CLIENT2_SECRET":"foobared",
-"QUAY_CLIENT_ID":"foobared",
-"QUAY_CLIENT_SECRET":"foobared",
+"AUTHORIZER_TYPE":"replaceme",
+"BD_CATALYST_SEVEN_BRIDGES_IMPORT_URL":"replaceme",
+"BD_CATALYST_TERRA_IMPORT_URL":"replaceme",
 "BITBUCKET_CLIENT_ID":"foobared",
 "BITBUCKET_CLIENT_SECRET":"foobared",
-"DOMAIN_NAME":"foobar",
-"HTTPS":false,
+"CHECK_URL_EXISTS_ENDPOINT":"replaceme",
+"CHECK_URL_LAMBDA_VERSION":"n/a",
 "COMPOSE_SETUP_VERSION":"foo",
-"DEPLOY_COMMIT_ID":"foo",
-"DISCOURSE_URL":"foobar",
-"DISCOURSE_KEY":"foobar",
+"CWL_PARSING_LAMBDA_VERSION":"n/a",
+"DATABASE_DOMAIN":"postgres",
+"DATABASE_GENERATED":false,
+"DEPLOY_VERSION":"replaceme",
 "DISCOURSE_CATEGORY_ID":"10",
+"DISCOURSE_KEY":"foobar",
+"DISCOURSE_URL":"foobar",
+"DOCKSTORE_DBPASSWORD":"replaceme",
+"DOCKSTORE_VERSION":"develop",
 "DOCUMENTATION_URL":"foobar",
+"DOMAIN_NAME":"foobar",
+"ELASTICSEARCH_DOMAIN":"replaceme",
+"ELASTICSEARCH_MAX_CONCURRENT_SESSIONS":"123",
+"ELASTICSEARCH_PASSWORD":"replaceme",
+"ELASTICSEARCH_PORT":"80",
+"ELASTICSEARCH_PROTOCOL":"http",
+"ELASTICSEARCH_USER":"replaceme",
+"ELWAZI_IMPORT_URL": "replaceme",
+"EXTERNAL_GOOGLE_CLIENT_PREFIX1":"replaceme",
 "FEATURED_CONTENT_URL":"foobar",
-"GITHUB_APP_PRIVATE_KEY_FILE": "/replaceme",
+"FEATURED_NEWS_URL":"replaceme",
+"GALAXY_PLUGIN_VERSION":"0.0.6",
 "GITHUB_APP_ID": "1234",
 "GITHUB_APP_NAME": "foobar",
-"TOOLTESTER_BUCKET_NAME": "replaceme",
-"CHECK_URL_EXISTS_ENDPOINT":"replaceme",
-"TAG_MANAGER_ID":"foobar",
+"GITHUB_APP_PRIVATE_KEY_FILE": "/replaceme",
+"GITHUB_CLIENT2_ID":"foobared",
+"GITHUB_CLIENT2_SECRET":"foobared",
 "GITLAB_CLIENT_ID":"foobar",
 "GITLAB_CLIENT_SECRET":"foobar",
 "GOOGLE_CLIENT_ID":"potato",
 "GOOGLE_CLIENT_SECRET":"potato",
+"HTTPS":false,
+"IS_FARGATE_DEPLOY":false,
 "LOGSTASH":false,
 "LOGSTASH_HOST":"replaceme",
-"TERRA_IMPORT_URL":"replaceme",
-"ELWAZI_IMPORT_URL": "replaceme",
+"NEXTFLOW_PARSING_LAMBDA_VERSION":"n/a",
 "BD_CATALYST_SEVEN_BRIDGES_IMPORT_URL":"replaceme",
 "BD_CATALYST_TERRA_IMPORT_URL":"replaceme",
 "AUTHORIZER_TYPE":"replaceme",
@@ -40,17 +51,21 @@
 "DATABASE_GENERATED":false,
 "ORCID_CLIENT_ID":"replaceme",
 "ORCID_CLIENT_SECRET":"replaceme",
+"ORCID_SCOPE":"replaceme",
 "ORCID_URL":"replaceme",
-"ZENODO_CLIENT_ID":"replaceme",
-"ZENODO_CLIENT_SECRET":"replaceme",
-"ZENODO_URL":"replaceme",
+"POSTGRES_DBPASSWORD":"replaceme",
+"PRODUCTION":false,
+"PUBLIC_LAUNCHER_IP_ADDRESS":"",
+"QUAY_CLIENT_ID":"foobared",
+"QUAY_CLIENT_SECRET":"foobared",
+"SAM_PATH":"replaceme",
 "SLACK_URL":"replaeceme",
-"GALAXY_PLUGIN_VERSION":"0.0.6",
-"DATABASE_DOMAIN":"postgres",
-"DBUSER":"dockstore",
-"DBPASSWORD":"dockstore",
-"CWL_PARSING_LAMBDA_VERSION":"n/a",
+"TAG_MANAGER_ID":"foobar",
+"TERRA_IMPORT_URL":"replaceme",
+"TOOLTESTER_BUCKET_NAME": "replaceme",
+"UI2_HASH":"develop",
 "WDL_PARSING_LAMBDA_VERSION":"n/a",
-"NEXTFLOW_PARSING_LAMBDA_VERSION":"n/a",
-"CHECK_URL_LAMBDA_VERSION":"n/a"
+"ZENODO_CLIENT_ID":"replaceme",
+"ZENODO_CLIENT_SECRET":"replaceme",
+"ZENODO_URL":"replaceme"
 }

install_bootstrap

@@ -20,15 +20,36 @@ MSG
 function template()
 {
     mkdir -p config
+
+    NGINX_CONF_DIRECTORY="config"
+    NGINX_HTML2_DIRECTORY="config"
+    WEBSERVICE_DIRECTORY="config"
+
+    if [ $IS_FARGATE_DEPLOY == "true" ]; then
+        mkdir -p config/webservice
+        mkdir -p config/nginx-conf
+        mkdir -p config/nginx-html2
+        # Place the config files into specific directories so they can be mounted to container paths as bind mounts
+        NGINX_CONF_DIRECTORY="${NGINX_CONF_DIRECTORY}/nginx-conf"
+        NGINX_HTML2_DIRECTORY="${NGINX_HTML2_DIRECTORY}/nginx-html2"
+        WEBSERVICE_DIRECTORY="${WEBSERVICE_DIRECTORY}/webservice"
+
+        wget -qO ${NGINX_HTML2_DIRECTORY}/index.html https://gui.dockstore.org/${UI2_HASH}/index.html
+        wget -qO ${NGINX_HTML2_DIRECTORY}/manifest.json https://gui.dockstore.org/${UI2_HASH}/manifest.json
+    fi
+
     mustache dockstore_launcher_config/compose.config templates/Dockerfile_ui2.template > config/Dockerfile_ui2
-    mustache dockstore_launcher_config/compose.config templates/robots.txt.template > config/robots.txt
-    mustache dockstore_launcher_config/compose.config templates/web.yml.template > config/web.yml
-    mustache dockstore_launcher_config/compose.config templates/default.nginx_http.conf.template > config/default.nginx_http.conf
-    mustache dockstore_launcher_config/compose.config templates/default.nginx_http.shared.conf.template > config/default.nginx_http.shared.conf
-    mustache dockstore_launcher_config/compose.config templates/default.nginx_http.security.conf.template > config/default.nginx_http.security.conf
-
-    mustache dockstore_launcher_config/compose.config templates/init_webservice.sh.template > config/init_webservice.sh
-    mustache dockstore_launcher_config/compose.config templates/init_migration.sh.template > config/init_migration.sh
+    mustache dockstore_launcher_config/compose.config templates/robots.txt.template > ${NGINX_HTML2_DIRECTORY}/robots.txt
+    mustache dockstore_launcher_config/compose.config templates/default.nginx_http.conf.template > ${NGINX_CONF_DIRECTORY}/default.nginx_http.conf
+    mustache dockstore_launcher_config/compose.config templates/default.nginx_http.shared.conf.template > ${NGINX_CONF_DIRECTORY}/default.nginx_http.shared.conf
+    mustache dockstore_launcher_config/compose.config templates/default.nginx_http.security.conf.template > ${NGINX_CONF_DIRECTORY}/default.nginx_http.security.conf
+
+    mustache dockstore_launcher_config/compose.config templates/web.yml.template > ${WEBSERVICE_DIRECTORY}/web.yml
+    mustache dockstore_launcher_config/compose.config templates/init_webservice.sh.template > ${WEBSERVICE_DIRECTORY}/init_webservice.sh
+    mustache dockstore_launcher_config/compose.config templates/init_migration.sh.template > ${WEBSERVICE_DIRECTORY}/init_migration.sh
+    chmod a+rx ${WEBSERVICE_DIRECTORY}/init_webservice.sh
+    chmod a+rx ${WEBSERVICE_DIRECTORY}/init_migration.sh
+
     mustache dockstore_launcher_config/compose.config templates/elasticsearch.yml > config/elasticsearch.yml
     mustache dockstore_launcher_config/compose.config templates/metricbeat.yml > config/metricbeat.yml
     mustache dockstore_launcher_config/compose.config templates/essnapshot_backup.sh > scripts/essnapshot_backup.sh
@@ -66,11 +87,13 @@ template
 
 download_galaxy "$*"
 
-# We need to set the environment variable for the image digest
-source .env
-DOCKSTORE_IMAGE_DIGEST=$(scripts/webservice-image-digest.py $DOCKSTORE_VERSION)
-echo "DOCKSTORE_IMAGE_DIGEST=$DOCKSTORE_IMAGE_DIGEST" >> .env
+if [ $IS_FARGATE_DEPLOY == "false" ]; then
+    # We need to set the environment variable for the image digest
+    source .env
+    DOCKSTORE_IMAGE_DIGEST=$(scripts/webservice-image-digest.py $DOCKSTORE_VERSION)
+    echo "DOCKSTORE_IMAGE_DIGEST=$DOCKSTORE_IMAGE_DIGEST" >> .env
 
-docker-compose build
+    docker-compose build
+fi
 
 echo "Exiting now."

scripts/tests/test-webservice-image-digest.py

@@ -16,6 +16,7 @@
 
 base_command = "python {}".format(script_location)
 branch = "develop"
+full_directory = "develop-b667562"
 simple_tag = "digest_test"
 annotated_tag = "1.12.0-beta.1"
 
@@ -28,6 +29,12 @@ class TestDigest(unittest.TestCase):
 #        self.assertEqual(ret, "sha256:52cf6b09e89a238bfd1d98dd01139442d67fcaaa377c179f315dd06555f7bcae")
 #        pass
 
+    def test_full_directory(self):
+        cmd = "{} {}".format(base_command, full_directory)
+        ret = subprocess.check_output(cmd, shell=True, universal_newlines=True).rstrip()
+        self.assertEqual(ret, "sha256:08c67131daf6109fadb19d994d753ede7ae28e41c675322e2980327597bcb665")
+        pass
+
     def test_simple_tag(self):
         cmd = "{} {}".format(base_command, simple_tag)
         ret = subprocess.check_output(cmd, shell=True, universal_newlines=True).rstrip()

scripts/webservice-image-digest.py

@@ -6,15 +6,24 @@
 images by digest as opposed to by tag and will also guarantee the image pulled
 from Quay has not been changed since being built.
 
+Assets in S3 follow the directory structure: `./branch-shortcommit/image-digest.txt`
+
+Provide a git tag, branch, or branch-shorthash
+
+The output is formatted to be easily used to select a specific image digest via docker
+
+sha256:08c67131daf6109fadb19d994d753ede7ae28e41c675322e2980327597bcb665
+
 """
 
 import argparse
 import requests
+import string
 
 parser = argparse.ArgumentParser(
   description='Gather an image digest for the Dockstore Webservice from S3 as created by CircleCI')
 parser.add_argument('tag', type=str,
-                    help='The git tag (or branch)')
+                    help='The git tag, branch, or branch-hash of a Webservice commit')
 
 args = parser.parse_args()
 
@@ -46,20 +55,26 @@ def get_commit_from_github(tag_or_branch):
   print("No commit for that tag or branch found!")
   exit(1)
 
-def get_digest_from_s3(tag, commit):
+def get_digest_from_s3(directory):
   # downloads the image-digest.txt from a directory in S3
   base_url = "https://gui.dockstore.org"
-  response = requests.get("{}/{}-{}/image-digest.txt".format(base_url, tag, commit[0:7]))
+  response = requests.get("{}/{}/image-digest.txt".format(base_url, directory))
   if (response.status_code != 200):
-    print("Expected a file at {}".format("{}/{}-{}/image-digest.txt".format(base_url, tag, commit[0:7])))
+    print("Expected a file at {}".format("{}/{}/image-digest.txt".format(base_url, directory)))
     print("The image-digest.txt was not found in S3, did the build succeed?")
     exit(1)
   # There is a newline at the end of the file we rstrip
   return response.text.rstrip()
 
 if __name__ == "__main__":
   # slashes are replaced with _ in docker image tags
-  commit = get_commit_from_github(args.tag)
-  circle_digest = get_digest_from_s3(args.tag, commit)
+  # check to see if input includes a dash followed by 7 chars
+  parsed = args.tag.split('-')
+  if len(parsed) == 2 and len(parsed[1]) == 7 and all(c in string.hexdigits for c in parsed[1]):
+    directory = args.tag
+  else:
+    commit = get_commit_from_github(args.tag)
+    directory = "{}-{}".format(args.tag, commit[0:7])
+  circle_digest = get_digest_from_s3(directory)
   print("sha256:{}".format(circle_digest))
   exit(0)

templates/default.nginx_http.conf.template

@@ -26,7 +26,12 @@ access_log  off;
 resolver 127.0.0.11 valid=10s;
 
 server {
+    {{#IS_FARGATE_DEPLOY}}
+    set $webservice "127.0.0.1";
+    {{/IS_FARGATE_DEPLOY}}
+    {{^IS_FARGATE_DEPLOY}}
     set $webservice "webservice";
+    {{/IS_FARGATE_DEPLOY}}
     server_name  .{{ DOMAIN_NAME }};
     include /etc/nginx/conf.d/default.nginx_http.shared.conf;
     include /etc/nginx/conf.d/default.nginx_http.security.conf;
@@ -102,7 +107,12 @@ server {
 
 # Server block for access via IP instead of domain
 server {
+    {{#IS_FARGATE_DEPLOY}}
+    set $webservice "127.0.0.1";
+    {{/IS_FARGATE_DEPLOY}}
+    {{^IS_FARGATE_DEPLOY}}
     set $webservice "webservice";
+    {{/IS_FARGATE_DEPLOY}}
     access_log  /var/log/nginx/access.log  custom;
     listen 4200 default_server;
 

templates/default.nginx_http.security.conf.template

@@ -18,7 +18,7 @@ add_header X-XSS-Protection "1; mode=block" always;
 add_header Referrer-Policy "strict-origin-when-cross-origin" always;
 
 # Explicitly list domains allowed to serve content for this site
-add_header Content-Security-Policy-Report-Only "report-uri https://api.dockstore-security.org/csp-report; default-src 'self'; object-src 'none'; base-uri 'self'; manifest-src 'self'; media-src 'self'; worker-src 'none'; script-src 'report-sample' 'self' 'unsafe-hashes' 'unsafe-inline' 'unsafe-eval' discuss.dockstore.org gui.dockstore.org *.twitter.com *.twimg.com www.google-analytics.com www.googletagmanager.com; style-src 'report-sample' 'self' 'unsafe-inline' cdnjs.cloudflare.com fonts.googleapis.com *.twitter.com *.twimg.com gui.dockstore.org; connect-src 'self' s3.amazonaws.com api.github.com view.commonwl.org www.google-analytics.com gui.dockstore.org; font-src 'self' fonts.gstatic.com gui.dockstore.org; frame-src 'self' discuss.dockstore.org platform.twitter.com; img-src data: 'self' avatars0.githubusercontent.com avatars1.githubusercontent.com avatars2.githubusercontent.com avatars3.githubusercontent.com camo.githubusercontent.com gui.dockstore.org i.imgur.com api.travis-ci.com img.shields.io quay.io via.placeholder.com *.wp.com *.googleusercontent.com www.googletagmanager.com www.google-analytics.com www.gravatar.com *.twitter.com *.twimg.com;" always;
+add_header Content-Security-Policy-Report-Only "report-uri https://api.dockstore-security.org/csp-report; default-src 'self'; object-src 'none'; base-uri 'self'; manifest-src 'self' dockstore.org; media-src 'self'; worker-src 'none'; script-src 'report-sample' 'self' 'unsafe-hashes' 'unsafe-inline' 'unsafe-eval' discuss.dockstore.org gui.dockstore.org *.twitter.com *.twimg.com www.google-analytics.com www.googletagmanager.com; style-src 'report-sample' 'self' 'unsafe-inline' cdnjs.cloudflare.com fonts.googleapis.com *.twitter.com *.twimg.com gui.dockstore.org; connect-src 'self' s3.amazonaws.com api.github.com view.commonwl.org www.google-analytics.com content.dockstore.org gui.dockstore.org; font-src 'self' fonts.gstatic.com gui.dockstore.org; frame-src 'self' discuss.dockstore.org platform.twitter.com youtube.com; img-src data: 'self' user-images.githubusercontent.com avatars.githubusercontent.com avatars0.githubusercontent.com avatars1.githubusercontent.com avatars2.githubusercontent.com avatars3.githubusercontent.com camo.githubusercontent.com circleci.com gui.dockstore.org gstatic.com i.imgur.com api.travis-ci.com travis-ci.com img.shields.io quay.io via.placeholder.com *.wp.com *.googleusercontent.com www.googletagmanager.com www.google-analytics.com www.gravatar.com *.twitter.com i.ytimg.com *.twimg.com zenodo.org;" always;
 
 # Hide server header
 proxy_hide_header Server;