Merge pull request #139 from icanbwell/updates_helix_spark_image_version

Updated helix.spark image version to latest 3.5.1.5, and other Spark pkgs

Author: changsu-bWell

Makefile

@@ -3,22 +3,24 @@ LANG=en_US.utf-8
 export LANG
 
 Pipfile.lock: Pipfile
-	docker compose run --rm --name sam dev sh -c "rm -f Pipfile.lock && pipenv lock --dev"
+	docker compose run --rm --name sam_fhir dev sh -c "rm -f Pipfile.lock && pipenv lock --dev"
 
 .PHONY:devdocker
 devdocker: ## Builds the docker for dev
-	docker-compose build
+	docker compose build --no-cache
 
 .PHONY:init
 init: devdocker up setup-pre-commit  ## Initializes the local developer environment
 
 .PHONY: up
 up: Pipfile.lock
-	docker compose up --build -d --remove-orphans
+	docker compose up --build -d
 
 .PHONY: down
-down:
-	docker compose down
+down: ## Brings down all the services in docker-compose
+	export DOCKER_CLIENT_TIMEOUT=300 && export COMPOSE_HTTP_TIMEOUT=300
+	docker compose down --remove-orphans && \
+	docker system prune -f
 
 .PHONY:clean-pre-commit
 clean-pre-commit: ## removes pre-commit hook
@@ -34,13 +36,13 @@ run-pre-commit: setup-pre-commit
 
 .PHONY:update
 update: down Pipfile.lock setup-pre-commit  ## Updates all the packages using Pipfile
-	docker compose run --rm --name spf_pipenv dev pipenv sync --dev && \
-	make devdocker && \
-	make pipenv-setup
+	docker compose run --rm --name sam_fhir_pipenv dev pipenv sync --dev && \
+	make pipenv-setup && \
+	make devdocker
 
 .PHONY:tests
 tests: up
-	docker compose run --rm --name sam_fhir dev pytest tests
+	docker compose run --rm --name sam_fhir_tests dev pytest tests
 
 .PHONY:continuous_integration
 continuous_integration: run-pre-commit
@@ -67,13 +69,13 @@ classes:
 
 .PHONY:pipenv-setup
 pipenv-setup:devdocker ## Run pipenv-setup to update setup.py with latest dependencies
-	docker compose run --rm --name spark_pipeline_framework dev sh -c "pipenv run pipenv install --skip-lock --categories \"pipenvsetup\" && pipenv run pipenv-setup sync --pipfile" && \
+	docker compose run --rm --name sam_fhir dev sh -c "pipenv run pipenv install --skip-lock --categories \"pipenvsetup\" && pipenv run pipenv-setup sync --pipfile" && \
 	make run-pre-commit
 
 
 .PHONY:shell
 shell:devdocker ## Brings up the bash shell in dev docker
-	docker compose run --rm --name sam_shell dev /bin/bash
+	docker compose run --rm --name sam_fhir_shell dev /bin/bash
 
 .PHONY:build
 build: ## Builds the docker for dev

Pipfile

@@ -8,8 +8,8 @@ name = "pypi"
 py4j="==0.10.9.7" # https://spark.apache.org/docs/latest/api/python/getting_started/install.html#dependencies
 # pyspark is needed for working with Spark
 pyspark="==3.5.1"  # should match the version of spark we use for testing
-sparkautomapper = ">=3.0.0"
-sparkfhirschemas = ">=2.0.0"
+sparkautomapper = ">=3.0.1"
+sparkfhirschemas = ">=2.0.2"
 deprecated = "*"
 
 [dev-packages]
@@ -31,15 +31,21 @@ pytest = ">=8.2.2"
 black = ">=24.4.2"
 # importlib-metadata is needed for working with metadata
 importlib-metadata = ">=5.2.0"
+# Sphinx is needed for generating documentation
 Sphinx="==4.1.2"
-sphinx-autoapi="==1.8.4"
-sphinx-rtd-theme="==0.5.2"
-myst-parser="==0.15.1"
+# sphinx-autoapi is needed for generating API documentation
+sphinx-autoapi="==2.0.0"
+# sphinx-rtd-theme is needed for the Read the Docs theme
+sphinx-rtd-theme="==1.0.0"
+# myst-parser is needed for parsing Markdown
+myst-parser="==0.17.2"
+
 xmlschema=">=1.6.4"
 jinja2=">=3.0.3"
 pycln=">=2.4.0"
 lxml = ">=4.6.3"
-sparkdataframecomparer = "==2.0.4"
+
+sparkdataframecomparer = "==2.0.6"
 
 # These dependencies are required for pipenv-setup.  They conflict with ones above, so we install these
 # only when running pipenv-setup

Pipfile.lock

@@ -32,8 +32,8 @@
     install_requires=[
         "py4j==0.10.9.7",
         "pyspark==3.5.1",
-        "sparkautomapper>=3.0.0",
-        "sparkfhirschemas>=2.0.0",
+        "sparkautomapper>=3.0.1",
+        "sparkfhirschemas>=2.0.2",
         "deprecated",
     ],
     classifiers=[

setup.py

@@ -1,4 +1,4 @@
-FROM imranq2/helix.spark:3.5.1.3-slim
+FROM imranq2/helix.spark:3.5.1.5-slim
 # https://github.com/icanbwell/helix.spark
 USER root