upgrading to python3.7

Author: Naor Livne

.travis.yml

@@ -1,6 +1,7 @@
 language: python
+dist: xenial
 python:
-  - "2.7"
+  - "3.7"
 install:
   - pip install -r requirements.txt
 script:

CONTRIBUTING

@@ -7,7 +7,7 @@ can help
 
 * Fork the latest branch of the component you want to contribute to
 * Make sure you have a [GitHub account](https://github.com/signup/free)
-* Use virtualenv to install all requirements from the requirements.txt file, this will require having GCC compiler, python2-dev & linux-headers depending on your environment setup due to psutil requiring compilation on some OS.
+* Use virtualenv to install all requirements from the requirements.txt file, this will require having GCC compiler, python3-dev & linux-headers depending on your environment setup due to psutil requiring compilation on some OS.
 * Fix the issue you want \ add a feature
 * Create a pull request
 

Dockerfile

@@ -1,11 +1,11 @@
 # it's offical so i'm using it + alpine so damn small
-FROM python:2.7.15-alpine3.9
+FROM python:3.7.2-alpine3.9
 
 # copy the codebase
 COPY . /worker
 
 # install required packages - requires build-base due to psutil GCC complier requirements
-RUN apk add --no-cache build-base python2-dev linux-headers
+RUN apk add --no-cache build-base python3-dev linux-headers
 RUN pip install -r /worker/requirements.txt
 
 #set python to be unbuffered

functions/docker_engine/docker_engine.py

@@ -25,15 +25,15 @@ def list_containers(self, app_name=""):
             try:
                 return self.cli.containers(filters={"label": "orchestrator=nebula"})
             except Exception as e:
-                print >> sys.stderr, e
+                print(e, file=sys.stderr)
                 print("failed getting list of all containers")
                 os._exit(2)
         else:
             try:
                 app_label = "app_name=" + app_name
                 return self.cli.containers(filters={"label": [app_label, "orchestrator=nebula"]}, all=True)
             except Exception as e:
-                print >> sys.stderr, e
+                print(e, file=sys.stderr)
                 print("failed getting list of containers where label is app_name=" + app_name)
                 os._exit(2)
 
@@ -59,7 +59,7 @@ def check_container_healthy(self, container_id):
         # it's health as a non existing container who's status is non existing is in the require state and therefor
         # can be considered healthy.
         except Exception as e:
-            print >> sys.stderr, e
+            print(e, file=sys.stderr)
             print("failed getting health status of container " + container_id)
             container_healthy = True
         return container_healthy
@@ -70,9 +70,9 @@ def registry_login(self, registry_user=None, registry_pass=None, registry_host="
                 registry_pass != "skip":
             print("logging in to registry")
             try:
-                print self.cli.login(username=registry_user, password=registry_pass, registry=registry_host)
+                print(self.cli.login(username=registry_user, password=registry_pass, registry=registry_host))
             except Exception as e:
-                print >> sys.stderr, e
+                print(e, file=sys.stderr)
                 print("problem logging into registry")
                 os._exit(2)
         else:
@@ -82,21 +82,21 @@ def registry_login(self, registry_user=None, registry_pass=None, registry_host="
     def pull_image(self, image_name, version_tag="latest"):
         print("pulling image " + image_name + ":" + str(version_tag))
         try:
-            print image_name
+            print(image_name)
             for line in self.cli.pull(image_name, str(version_tag), stream=True):
                 print(json.dumps(json.loads(line), indent=4))
         except Exception as e:
-            print >> sys.stderr, e
+            print(e, file=sys.stderr)
             print("problem pulling image " + image_name + ":" + str(version_tag))
             os._exit(2)
 
     # prune unused images
     def prune_images(self):
         print("pruning unused images")
         try:
-            print self.cli.prune_images()
+            print(self.cli.prune_images())
         except Exception as e:
-            print >> sys.stderr, e
+            print(e, file=sys.stderr)
             print("problem pruning unused image")
             os._exit(2)
 
@@ -111,16 +111,16 @@ def create_container(self, app_name, container_name, image_name, host_configurat
                                                                                          "orchestrator": "nebula"},
                                                           networking_config=self.create_networking_config(
                                                               default_network))
-            print("successfully created container " + container_name)
+            print(("successfully created container " + container_name))
             return container_created
         except Exception as e:
-            print >> sys.stderr, e
+            print(e, file=sys.stderr)
             print("failed creating container " + container_name)
             os._exit(2)
 
     # stop container, default timeout set to 5 seconds, will try to kill if stop failed
     def stop_container(self, container_name, stop_timout=5):
-        print("stopping container " + container_name)
+        print(("stopping container " + container_name))
         try:
             reply = self.cli.stop(container_name, stop_timout)
             return reply
@@ -130,46 +130,46 @@ def stop_container(self, container_name, stop_timout=5):
                 time.sleep(3)
                 return reply
             except Exception as e:
-                print >> sys.stderr, e
+                print(e, file=sys.stderr)
                 print("problem stopping container " + container_name)
                 os._exit(2)
 
     # start container
     def start_container(self, container_name):
-        print("starting container " + container_name)
+        print(("starting container " + container_name))
         try:
             return self.cli.start(container_name)
         except "APIError" as e:
-            print >> sys.stderr, e
+            print(e, file=sys.stderr)
             print("problem starting container - most likely port bind already taken")
         except not "APIError" as e:
-            print >> sys.stderr, e
+            print(e, file=sys.stderr)
             print("problem starting container " + container_name)
             os._exit(2)
 
     # restart container, default timeout set to 2 seconds
     def restart_container(self, container_name, stop_timout=2):
-        print("restarting container " + container_name)
+        print(("restarting container " + container_name))
         try:
             return self.cli.restart(container_name, stop_timout)
         except "APIError" as e:
-            print >> sys.stderr, e
+            print(e, file=sys.stderr)
             print("problem starting container - most likely port bind already taken")
         except not "APIError" as e:
-            print >> sys.stderr, e
+            print(e, file=sys.stderr)
             print("problem restarting container " + container_name)
             os._exit(2)
 
     # remove container
     def remove_container(self, container_name):
-        print("removing container " + container_name)
+        print(("removing container " + container_name))
         try:
             return self.cli.remove_container(container_name)
         except:
             try:
                 return self.cli.remove_container(container_name, force=True)
             except Exception as e:
-                print >> sys.stderr, e
+                print(e, file=sys.stderr)
                 print("problem removing container " + container_name)
             os._exit(2)
 
@@ -180,7 +180,7 @@ def create_container_host_config(self, port_binds, volumes, devices, privileged,
                                                binds=volumes, devices=devices, privileged=privileged,
                                                network_mode=network_mode)
         except Exception as e:
-            print >> sys.stderr, e
+            print(e, file=sys.stderr)
             print("problem creating host config")
             os._exit(2)
 
@@ -194,7 +194,7 @@ def create_networking_config(self, starting_network=""):
             )
             return networking_config
         except Exception as e:
-            print >> sys.stderr, e
+            print(e, file=sys.stderr)
             print("problem creating network config")
             os._exit(2)
 
@@ -203,7 +203,7 @@ def connect_to_network(self, container, net_id):
         try:
             self.cli.connect_container_to_network(container, net_id)
         except Exception as e:
-            print >> sys.stderr, e
+            print(e, file=sys.stderr)
             print("problem connecting to network " + net_id)
             os._exit(2)
 
@@ -244,7 +244,7 @@ def run_container(self, app_name, container_name, image_name, bind_port, ports,
                 try:
                     self.connect_to_network(container_name, self.get_net_id(network))
                 except Exception as e:
-                    print >> sys.stderr, e
+                    print(e, file=sys.stderr)
                     print("problem connecting to network " + network)
                     os._exit(2)
 

functions/misc/server.py

@@ -7,7 +7,7 @@ def get_number_of_cpu_cores():
         cpu_number = multiprocessing.cpu_count()
         return cpu_number
     except Exception as e:
-        print >> sys.stderr, e
+        print(e, file=sys.stderr)
         print("error getting the number of cpu core")
         os._exit(2)
 
@@ -19,6 +19,6 @@ def get_total_memory_size_in_mb():
         total_memory_in_mb = int(memory_in_bytes.total / 1024 / 1024)
         return total_memory_in_mb
     except Exception as e:
-        print >> sys.stderr, e
+        print(e, file=sys.stderr)
         print("error getting the number of cpu core")
         os._exit(2)

shippable.yml

@@ -1,7 +1,7 @@
 language: python
 
 python:
-  - 2.7
+  - 3.7
 
 integrations:
   hub:

worker.py

@@ -14,13 +14,13 @@ def get_conf_setting(setting, settings_json, default_value="skip"):
     try:
         setting_value = os.getenv(setting.upper(), settings_json.get(setting, default_value))
     except Exception as e:
-        print >> sys.stderr, e
-        print >> sys.stderr, "missing " + setting + " config setting"
-        print("missing " + setting + " config setting")
+        print(e, file=sys.stderr)
+        print("missing " + setting + " config setting", file=sys.stderr)
+        print(("missing " + setting + " config setting"))
         os._exit(2)
     if setting_value == "skip":
-        print >> sys.stderr, "missing " + setting + " config setting"
-        print("missing " + setting + " config setting")
+        print("missing " + setting + " config setting", file=sys.stderr)
+        print(("missing " + setting + " config setting"))
         os._exit(2)
     return setting_value
 
@@ -79,7 +79,7 @@ def roll_containers(app_json, force_pull=True):
                     port_binds[x] = x + idx
                     port_list.append(x)
                 elif isinstance(x, dict):
-                    for host_port, container_port in x.iteritems():
+                    for host_port, container_port in x.items():
                         port_binds[int(container_port)] = int(host_port) + idx
                         port_list.append(container_port)
                 else:
@@ -132,7 +132,7 @@ def start_containers(app_json, force_pull=True):
                     port_binds[x] = x + container_number - 1
                     port_list.append(x)
                 elif isinstance(x, dict):
-                    for host_port, container_port in x.iteritems():
+                    for host_port, container_port in x.items():
                         port_binds[int(container_port)] = int(host_port) + container_number - 1
                         port_list.append(container_port)
                 else:
@@ -153,7 +153,7 @@ def start_containers(app_json, force_pull=True):
 
 # figure out how many containers are needed
 def containers_required(app_json):
-    for scale_type, scale_amount in app_json["containers_per"].iteritems():
+    for scale_type, scale_amount in app_json["containers_per"].items():
         if scale_type == "cpu":
             containers_needed = int(cpu_cores * scale_amount)
         elif scale_type == "memory" or scale_type == "mem":
@@ -178,7 +178,7 @@ def restart_unhealthy_containers():
                 if docker_socket.check_container_healthy(nebula_container["Id"]) is False:
                     docker_socket.restart_container(nebula_container["Id"])
     except Exception as e:
-        print >> sys.stderr, e
+        print(e, file=sys.stderr)
         print("failed checking containers health")
         os._exit(2)
 
@@ -245,7 +245,7 @@ def get_device_group_info(nebula_connection_object, device_group_to_get_info):
                 print("nebula manager initial connection check failure, dropping container")
                 os._exit(2)
         except Exception as e:
-            print >> sys.stderr, e
+            print(e, file=sys.stderr)
             print("error confirming connection to nebula manager - please check connection & authentication params and "
                   "that the manager is online")
             os._exit(2)
@@ -260,24 +260,24 @@ def get_device_group_info(nebula_connection_object, device_group_to_get_info):
         # make sure the device_group exists in the nebula cluster
         while local_device_group_info["status_code"] == 403 and \
                 local_device_group_info["reply"]["device_group_exists"] is False:
-            print("device_group " + device_group + " doesn't exist in nebula cluster, waiting for it to be created")
+            print(("device_group " + device_group + " doesn't exist in nebula cluster, waiting for it to be created"))
             local_device_group_info = get_device_group_info(nebula_connection, device_group)
             time.sleep(nebula_manager_check_in_time)
 
         # start all apps that are set to running on boot
         for nebula_app in local_device_group_info["reply"]["apps"]:
             if nebula_app["running"] is True:
-                print("initial start of " + nebula_app["app_name"] + " app")
+                print(("initial start of " + nebula_app["app_name"] + " app"))
                 start_containers(nebula_app)
-                print("completed initial start of " + nebula_app["app_name"] + " app")
+                print(("completed initial start of " + nebula_app["app_name"] + " app"))
 
         # open a thread which is in charge of restarting any containers which healthcheck shows them as unhealthy
         print("starting work container health checking thread")
         Thread(target=restart_unhealthy_containers).start()
 
         # loop forever
-        print("starting device_group " + device_group + " /info check loop, configured to check for changes every "
-              + str(nebula_manager_check_in_time) + " seconds")
+        print(("starting device_group " + device_group + " /info check loop, configured to check for changes every "
+              + str(nebula_manager_check_in_time) + " seconds"))
         while True:
 
             # wait the configurable time before checking the device_group info page again
@@ -296,20 +296,20 @@ def get_device_group_info(nebula_connection_object, device_group_to_get_info):
                     if remote_nebula_app["app_id"] > local_device_group_info["reply"]["apps"][local_app_index]["app_id"]:
                         monotonic_id_increase = True
                         if remote_nebula_app["running"] is False:
-                            print("stopping app " + remote_nebula_app["app_name"] +
-                                  " do to changes in the app configuration")
+                            print(("stopping app " + remote_nebula_app["app_name"] +
+                                  " do to changes in the app configuration"))
                             stop_containers(remote_nebula_app)
                         elif remote_nebula_app["rolling_restart"] is True and \
                                 local_device_group_info["reply"]["apps"][local_app_index]["running"] is True:
-                            print("rolling app " + remote_nebula_app["app_name"] +
-                                  " do to changes in the app configuration")
+                            print(("rolling app " + remote_nebula_app["app_name"] +
+                                  " do to changes in the app configuration"))
                             roll_containers(remote_nebula_app)
                         else:
-                            print("restarting app " + remote_nebula_app["app_name"] +
-                                  " do to changes in the app configuration")
+                            print(("restarting app " + remote_nebula_app["app_name"] +
+                                  " do to changes in the app configuration"))
                             restart_containers(remote_nebula_app)
                 else:
-                    print("restarting app " + remote_nebula_app["app_name"] + " do to changes in the app configuration")
+                    print(("restarting app " + remote_nebula_app["app_name"] + " do to changes in the app configuration"))
                     monotonic_id_increase = True
                     restart_containers(remote_nebula_app)
 
@@ -318,8 +318,8 @@ def get_device_group_info(nebula_connection_object, device_group_to_get_info):
                 monotonic_id_increase = True
                 for local_nebula_app in local_device_group_info["reply"]["apps"]:
                     if local_nebula_app["app_name"] not in remote_device_group_info["reply"]["apps_list"]:
-                        print("removing app " + local_nebula_app["app_name"] +
-                              " do to changes in the app configuration")
+                        print(("removing app " + local_nebula_app["app_name"] +
+                              " do to changes in the app configuration"))
                         stop_containers(local_nebula_app)
 
             # logic that runs image pruning if prune_id increased
@@ -333,6 +333,6 @@ def get_device_group_info(nebula_connection_object, device_group_to_get_info):
                 local_device_group_info = remote_device_group_info
 
     except Exception as e:
-        print >> sys.stderr, e
+        print(e, file=sys.stderr)
         print("failed main loop - exiting")
         os._exit(2)