Merge pull request #1593 from namark/hash-checks-in-python-build-scripts

Added hash checks for downloaded items in python build scripts.

Author: digisomni

hifi_qt.py

@@ -134,8 +134,10 @@ def __init__(self, args):
 
         if 'Windows' == system:
             self.qtUrl = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_WIN_URLS').split(";")
+            self.qtSha512 = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_WIN_SHA512')
         elif 'Darwin' == system:
             self.qtUrl = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_MAC_URLS').split(";")
+            self.qtSha512 = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_MAC_SHA512')
         elif 'Linux' == system:
             import distro
             cpu_architecture = platform.machine()
@@ -149,6 +151,7 @@ def __init__(self, args):
                 if distro.id() == 'ubuntu' or distro.id() == 'linuxmint':
                     if (distro.id() == 'ubuntu' and u_major == 18) or distro.id() == 'linuxmint' and u_major == 19:
                         self.qtUrl = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_LINUX_URLS').split(";")
+                        self.qtSha512 = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_LINUX_SHA512')
                     elif (distro.id() == 'ubuntu' and u_major > 18) or (distro.id() == 'linuxmint' and u_major > 19):
                         self.__no_qt_package_error()
                     else:
@@ -166,6 +169,7 @@ def __init__(self, args):
 
                     if u_major == 18:
                         self.qtUrl = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_UBUNTU_AARCH64_URLS').split(";")
+                        self.qtSha512 = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_UBUNTU_AARCH64_SHA512')
                     elif u_major > 19:
                         self.__no_qt_package_error()
                     else:
@@ -179,6 +183,7 @@ def __init__(self, args):
 
                     if u_major == 10:
                         self.qtUrl = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_DEBIAN_AARCH64_URLS').split(";")
+                        self.qtSha512 = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_DEBIAN_AARCH64_SHA512')
                     elif u_major > 10:
                         self.__no_qt_package_error()
                     else:
@@ -213,7 +218,7 @@ def writeConfig(self):
     def installQt(self):
         if not os.path.isdir(self.fullPath):
             print("Fetching Qt from {} to {}".format(self.qtUrl, self.path))
-            hifi_utils.downloadAndExtract(self.qtUrl, self.path)
+            hifi_utils.downloadAndExtract(self.qtUrl, self.path, self.qtSha512)
         else:
             print ('Qt has already been downloaded')
 

hifi_utils.py

@@ -80,7 +80,8 @@ def executeSubprocess(processArgs, folder=None, env=None):
         os.chdir(restoreDir)
 
 
-def hashFile(file, hasher = hashlib.sha512()):
+def hashFile(file, hasherType = hashlib.sha512):
+    hasher = hasherType()
     with open(file, "rb") as f:
         for chunk in iter(lambda: f.read(4096), b""):
             hasher.update(chunk)
@@ -99,7 +100,7 @@ def hashFolder(folder):
     filenames = recursiveFileList(folder)
     return hashFiles(filenames)
 
-def downloadFile(urls, hash=None, hasher=hashlib.sha512(), retries=3):
+def downloadFile(urls, hash=None, hasher=hashlib.sha512, retries=3):
     for url in urls:
         for i in range(retries):
             tempFileName = None
@@ -125,14 +126,15 @@ def downloadFile(urls, hash=None, hasher=hashlib.sha512(), retries=3):
             # Verify the hash
             if hash is not None and hash != downloadHash:
                 print("Try {}: Downloaded file {} hash {} does not match expected hash {} for url {}".format(i + 1, tempFileName, downloadHash, hash, url))
+                print("File stats: ", os.stat(tempFileName))
                 os.remove(tempFileName)
                 continue
             return tempFileName
 
     raise RuntimeError("Failed to download file from any of {}".format(urls))
 
 
-def downloadAndExtract(urls, destPath, hash=None, hasher=hashlib.sha512()):
+def downloadAndExtract(urls, destPath, hash=None, hasher=hashlib.sha512):
     tempFileName = downloadFile(urls, hash, hasher)
     try:
         with zipfile.ZipFile(tempFileName) as zip:

hifi_vcpkg.py

@@ -94,27 +94,33 @@ def __init__(self, args):
             self.exe = os.path.join(self.path, 'vcpkg.exe')
             self.bootstrapCmds = [ os.path.join(self.path, 'bootstrap-vcpkg.bat'), '-disableMetrics' ]
             self.vcpkgUrl = self.readVar('EXTERNAL_VCPKG_WIN_CLIENT_URLS').split(';')
+            self.vcpkgSha512 = self.readVar('EXTERNAL_VCPKG_WIN_CLIENT_SHA512')
             self.hostTriplet = 'x64-windows'
             if usePrebuilt:
                 self.prebuiltArchive = self.readVar('EXTERNAL_VCPKG_WIN_URLS').split(';')
+                self.prebuiltArchiveSha512 = self.readVar('EXTERNAL_VCPKG_WIN_SHA512')
         elif 'Darwin' == system:
             self.exe = os.path.join(self.path, 'vcpkg')
             self.bootstrapCmds = [ os.path.join(self.path, 'bootstrap-vcpkg.sh'), '--allowAppleClang', '-disableMetrics' ]
             self.vcpkgUrl = self.readVar('EXTERNAL_VCPKG_MAC_CLIENT_URLS').split(';')
+            self.vcpkgSha512 = self.readVar('EXTERNAL_VCPKG_MAC_CLIENT_SHA512')
             self.hostTriplet = 'x64-osx'
             # Potential fix for a vcpkg build issue on OSX (see https://github.com/microsoft/vcpkg/issues/9029)
             self.bootstrapEnv['CXXFLAGS'] = '-D_CTERMID_H_'
             if usePrebuilt:
                 self.prebuiltArchive = self.readVar('EXTERNAL_VCPKG_MAC_URLS').split(';')
+                self.prebuiltArchiveSha512 = self.readVar('EXTERNAL_VCPKG_MAC_SHA512')
         elif 'Linux' == system and 'aarch64' == machine:
             self.exe = os.path.join(self.path, 'vcpkg')
             self.bootstrapCmds = [ os.path.join(self.path, 'bootstrap-vcpkg.sh'), '-disableMetrics' ]
             self.vcpkgUrl = self.readVar('EXTERNAL_VCPKG_LINUX_AARCH64_URLS').split(';')
+            self.vcpkgSha512 = self.readVar('EXTERNAL_VCPKG_LINUX_AARCH64_SHA512')
             self.hostTriplet = 'arm64-linux'
         else:
             self.exe = os.path.join(self.path, 'vcpkg')
             self.bootstrapCmds = [ os.path.join(self.path, 'bootstrap-vcpkg.sh'), '-disableMetrics' ]
             self.vcpkgUrl = self.readVar('EXTERNAL_VCPKG_LINUX_CLIENT_URLS').split(';')
+            self.vcpkgSha512 = self.readVar('EXTERNAL_VCPKG_LINUX_CLIENT_SHA512')
             self.hostTriplet = 'x64-linux'
 
         if self.args.android:
@@ -202,7 +208,7 @@ def bootstrap(self):
                 hifi_utils.executeSubprocess(self.bootstrapCmds, folder=self.path, env=self.bootstrapEnv)
             else:
                 print("Fetching vcpkg from {} to {}".format(self.vcpkgUrl, self.path))
-                hifi_utils.downloadAndExtract(self.vcpkgUrl, self.path)
+                hifi_utils.downloadAndExtract(self.vcpkgUrl, self.path, self.vcpkgSha512)
 
         print("Replacing port files")
         portsPath = os.path.join(self.path, 'ports')
@@ -237,7 +243,7 @@ def setupDependencies(self, qt=None):
         if self.prebuiltArchive:
             if not os.path.isfile(self.prebuildTagFile):
                 print('Extracting ' + self.prebuiltArchive + ' to ' + self.path)
-                hifi_utils.downloadAndExtract(self.prebuiltArchive, self.path)
+                hifi_utils.downloadAndExtract(self.prebuiltArchive, self.path, self.prebuiltArchiveSha512)
                 self.writePrebuildTag()
             return
 
@@ -283,9 +289,8 @@ def setupAndroidDependencies(self):
             dest = os.path.join(self.path, 'installed')
             url = self.readVar('EXTERNAL_VCPKG_ANDROID_URLS').split(';')
             # FIXME I don't know why the hash check frequently fails here.  If you examine the file later it has the right hash
-            #hash = self.readVar(('EXTERNAL_VCPKG_ANDROID_SHA512')
-            #hifi_utils.downloadAndExtract(url, dest, hash)
-            hifi_utils.downloadAndExtract(url, dest)
+            hash = self.readVar('EXTERNAL_VCPKG_ANDROID_SHA512')
+            hifi_utils.downloadAndExtract(url, dest, hash)
 
         print("Installing additional android archives")
         androidPackages = hifi_android.getPlatformPackages()

prebuild.py

@@ -100,6 +100,7 @@ def main():
 
     args = parse_args()
     nsis_urls = hifi_utils.readEnviromentVariableFromFile(args.build_root, 'EXTERNAL_NSIS_HIFI_PLUGINS_URLS').split(';')
+    nsis_sha512 = hifi_utils.readEnviromentVariableFromFile(args.build_root, 'EXTERNAL_NSIS_HIFI_PLUGINS_SHA512')
 
     if args.ci_build:
         logging.basicConfig(datefmt='%H:%M:%S', format='%(asctime)s %(guid)s %(message)s', level=logging.INFO)
@@ -111,7 +112,7 @@ def main():
     if 'Windows' == system and 'CI_BUILD' in os.environ and os.environ["CI_BUILD"] == "Github":
         logger.info("Downloading NSIS")
         with timer('NSIS'):
-            hifi_utils.downloadAndExtract(nsis_urls, "C:/Program Files (x86)")
+            hifi_utils.downloadAndExtract(nsis_urls, "C:/Program Files (x86)", nsis_sha512)
 
     qtInstallPath = None
     # If not android, install our Qt build