From fc6da71339569fd1ea9b07b5da9795c7bf1e6d0c Mon Sep 17 00:00:00 2001 From: Javier Garcia Ordonez Date: Wed, 28 Aug 2024 16:24:28 +0200 Subject: [PATCH 01/11] change to root user --- {{cookiecutter.project_slug}}/.osparc/bin/ooil | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/{{cookiecutter.project_slug}}/.osparc/bin/ooil b/{{cookiecutter.project_slug}}/.osparc/bin/ooil index fb85469d..eb407b89 100755 --- a/{{cookiecutter.project_slug}}/.osparc/bin/ooil +++ b/{{cookiecutter.project_slug}}/.osparc/bin/ooil @@ -7,6 +7,7 @@ set -o pipefail IFS=$'\n\t' IMAGE_NAME="itisfoundation/service-integration:${OOIL_IMAGE_TAG:-master-github-latest}" +# IMAGE_NAME="itisfoundation/ci-service-integration-library:v1.0.4" ## TODO change to latest WORKDIR="$(pwd)" # @@ -23,7 +24,7 @@ run() { --pull=always \ --volume="/etc/group:/etc/group:ro" \ --volume="/etc/passwd:/etc/passwd:ro" \ - --user="$(id --user "$USER")":"$(id --group "$USER")" \ + --user root \ --volume "$WORKDIR":/src \ --workdir=/src \ "$IMAGE_NAME" \ From 998ae747ce39395a5e75c3f1c41375cf5d545f0d Mon Sep 17 00:00:00 2001 From: Javier Garcia Ordonez Date: Wed, 28 Aug 2024 16:25:31 +0200 Subject: [PATCH 02/11] remove push GitHub action --- .../.github/workflows/build-test-publish.yml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/{{cookiecutter.project_slug}}/.github/workflows/build-test-publish.yml b/{{cookiecutter.project_slug}}/.github/workflows/build-test-publish.yml index 71775729..0187e6c2 100644 --- a/{{cookiecutter.project_slug}}/.github/workflows/build-test-publish.yml +++ b/{{cookiecutter.project_slug}}/.github/workflows/build-test-publish.yml @@ -41,13 +41,13 @@ jobs: make info-build - name: test run: make tests - - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - name: push - run: | - ./.github/dockerhub_login.bash - make push - - if: github.event_name == 'push' && github.ref != 'refs/heads/master' - name: push - run: | - ./.github/dockerhub_login.bash - make push-version + # - if: github.event_name == 'push' && github.ref == 'refs/heads/master' + # name: push + # run: | + # ./.github/dockerhub_login.bash + # make push + # - if: github.event_name == 'push' && github.ref != 'refs/heads/master' + # name: push + # run: | + # ./.github/dockerhub_login.bash + # make push-version From 949bc8683cbc8de739869fc8f9f945294705da61 Mon Sep 17 00:00:00 2001 From: Javier Garcia Ordonez Date: Wed, 28 Aug 2024 17:10:39 +0200 Subject: [PATCH 03/11] changed .osparc and Makefile structure to that of cookiecutter-jupyter --- .../.github/workflows/check-image.yml | 24 +++ .../.osparc/Makefile | 68 -------- .../.osparc/bin/activate | 67 -------- .../.osparc/bin/ooil | 41 ----- {{cookiecutter.project_slug}}/.osparc/bin/yq | 17 -- .../.osparc/metadata.yml | 5 +- {{cookiecutter.project_slug}}/Makefile | 148 +++++------------- 7 files changed, 62 insertions(+), 308 deletions(-) create mode 100644 {{cookiecutter.project_slug}}/.github/workflows/check-image.yml delete mode 100644 {{cookiecutter.project_slug}}/.osparc/Makefile delete mode 100755 {{cookiecutter.project_slug}}/.osparc/bin/activate delete mode 100755 {{cookiecutter.project_slug}}/.osparc/bin/ooil delete mode 100755 {{cookiecutter.project_slug}}/.osparc/bin/yq diff --git a/{{cookiecutter.project_slug}}/.github/workflows/check-image.yml b/{{cookiecutter.project_slug}}/.github/workflows/check-image.yml new file mode 100644 index 00000000..0aed71b0 --- /dev/null +++ b/{{cookiecutter.project_slug}}/.github/workflows/check-image.yml @@ -0,0 +1,24 @@ +name: Build and check image + +on: [push, pull_request] + +jobs: + verify-image-build: + runs-on: ubuntu-latest + steps: + - name: Checkout repo content + uses: actions/checkout@v2 + - name: ooil version + uses: docker://itisfoundation/ci-service-integration-library:v1.0.4 + with: + args: ooil --version + - name: Assemble docker compose spec + uses: docker://itisfoundation/ci-service-integration-library:v1.0.4 + with: + args: ooil compose + - name: Build all images if multiple + uses: docker://itisfoundation/ci-service-integration-library:v1.0.4 + with: + args: docker compose build + - name: test + run: make tests diff --git a/{{cookiecutter.project_slug}}/.osparc/Makefile b/{{cookiecutter.project_slug}}/.osparc/Makefile deleted file mode 100644 index 57dbc757..00000000 --- a/{{cookiecutter.project_slug}}/.osparc/Makefile +++ /dev/null @@ -1,68 +0,0 @@ - -.DEFAULT_GOAL := help -SHELL := /bin/bash - - -REPO_DIR := $(abspath $(CURDIR)/..) - -# NOTE that IMAGES variable can change when docker-compose.yml gets rebuilt. Do NOT use := !!! -IMAGES = $(shell ./bin/yq eval '.services.*.image' docker-compose.yml) - -export DOCKER_REGISTRY ?= registry:5000 - - -.PHONY: info -info: ## lists environments and required tools - # tools - @echo 'docker : $(shell docker --version 2>/dev/null || echo ERROR required tool missing )' - @echo 'docker compose : $(shell docker compose version 2>/dev/null || echo ERROR required tool missing )' - @echo 'docker buildx : $(shell docker buildx version 2>/dev/null || echo WARNING optional tool missing )' - @echo 'make : $(shell make --version 2>&1 | head -n 1)' - @echo 'awk : $(shell awk -W version 2>&1 | head -n 1)' - @echo 'sed : $(shell sed --version 2>&1 | head -n 1)' - # environments - @echo "REPO_DIR = ${REPO_DIR}" - @echo "IMAGES = ${IMAGES}" - @echo "DOCKER_REGISTRY = ${DOCKER_REGISTRY}" - - -.PHONY: update-version -update-version: - cd $(REPO_DIR) \ - && . update_version.sh \ - && cd - - -.PHONY: compose -compose: update-version ## creates docker-compose.yml - # creating compose specs - cd $(REPO_DIR) \ - && .osparc/bin/ooil compose -f .osparc/docker-compose.yml - - -.PHONY: build build-nc -build build-nc: compose ## builds image. Suffix -nc disables cache - # building - docker compose build $(if $(findstring -nc,$@),--no-cache,) - - -.PHONY: push -push: ## retags and pushes to ${DOCKER_REGISTRY} - # tag & push - @for image in ${IMAGES}; do \ - echo "Tagging and pushing ${DOCKER_REGISTRY}/$$image ..."; \ - docker tag "$$image" "${DOCKER_REGISTRY}/$$image"; \ - docker push "${DOCKER_REGISTRY}/$$image"; \ - done - # registry view - @curl --silent ${DOCKER_REGISTRY}/v2/_catalog | jq - - - -.PHONY: all -all: compose build-nc push ## all workflow - @echo "'${IMAGES}' built, and pushed to '${DOCKER_REGISTRY}'" - - -.PHONY: help -help: ## help on rule's targets - @awk --posix 'BEGIN {FS = ":.*?## "} /^[[:alpha:][:space:]_-]+:.*?## / {printf "%-20s %s\n", $$1, $$2}' $(MAKEFILE_LIST) diff --git a/{{cookiecutter.project_slug}}/.osparc/bin/activate b/{{cookiecutter.project_slug}}/.osparc/bin/activate deleted file mode 100755 index 1bcfd52f..00000000 --- a/{{cookiecutter.project_slug}}/.osparc/bin/activate +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash - -# Store the current PATH -export _OLD_OSPARC_PATH=$PATH - - -# Get the directory of the currently running script -BIN_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -OSPARC_ENV=$(dirname "${BIN_DIR}") - - -# Add .osparc/bin to the PATH -export PATH="$BIN_DIR/.osparc/bin:$PATH" - - -deactivate () { - # Unset the OLD_PATH variable - # ! [ -z ${VAR+_} ] returns true if VAR is declared at all - if ! [ -z "${_OLD_OSPARC_PATH:+_}" ] ; then - PATH="$_OLD_OSPARC_PATH" - export PATH - unset _OLD_OSPARC_PATH - fi - - # The hash command must be called to get it to forget past - # commands. Without forgetting past commands the $PATH changes - # we made may not be respected - hash -r 2>/dev/null - - - # Removes (osparc) in prompt - if ! [ -z "${_OLD_OSPARC_PS1+_}" ] ; then - PS1="$_OLD_OSPARC_PS1" - export PS1 - unset _OLD_OSPARC_PS1 - fi - - unset OSPARC_ENV - unset OSPARC_ENV_PROMPT - if [ ! "${1-}" = "nondestructive" ] ; then - # Self destruct! - unset -f deactivate - echo "osparc environment deactivated" - fi -} - -# unset irrelevant variables -deactivate nondestructive - - -# Adds (osparc) in prompt -if [ "xosparc" != x ] ; then - OSPARC_ENV_PROMPT=".osparc" -else - OSPARC_ENV_PROMPT=$(basename "$OSPARC_ENV") -fi -export OSPARC_ENV_PROMPT - - -if [ -z "${OSPARC_ENV_DISABLE_PROMPT-}" ] ; then - _OLD_OSPARC_PS1="${PS1-}" - PS1="(${OSPARC_ENV_PROMPT}) ${PS1-}" - export PS1 -fi - -# Inform the user -echo "Environment activated. To deactivate, type 'deactivate'" diff --git a/{{cookiecutter.project_slug}}/.osparc/bin/ooil b/{{cookiecutter.project_slug}}/.osparc/bin/ooil deleted file mode 100755 index eb407b89..00000000 --- a/{{cookiecutter.project_slug}}/.osparc/bin/ooil +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# http://redsymbol.net/articles/unofficial-bash-strict-mode/ - -set -o errexit -set -o nounset -set -o pipefail -IFS=$'\n\t' - -IMAGE_NAME="itisfoundation/service-integration:${OOIL_IMAGE_TAG:-master-github-latest}" -# IMAGE_NAME="itisfoundation/ci-service-integration-library:v1.0.4" ## TODO change to latest -WORKDIR="$(pwd)" - -# -# NOTE: with --interactive --tty the command below will -# produce colors in the outputs. The problem is that -# . ooil >VERSION will insert special color codes -# . in the VERSION file which make it unusable as a variable -# . when cat VERSION !! -# - -run() { - docker run \ - --rm \ - --pull=always \ - --volume="/etc/group:/etc/group:ro" \ - --volume="/etc/passwd:/etc/passwd:ro" \ - --user root \ - --volume "$WORKDIR":/src \ - --workdir=/src \ - "$IMAGE_NAME" \ - "$@" -} - -# ---------------------------------------------------------------------- -# MAIN -# -# USAGE -# ooil --help - -run "$@" -# ---------------------------------------------------------------------- diff --git a/{{cookiecutter.project_slug}}/.osparc/bin/yq b/{{cookiecutter.project_slug}}/.osparc/bin/yq deleted file mode 100755 index 99b23150..00000000 --- a/{{cookiecutter.project_slug}}/.osparc/bin/yq +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -set -o errexit -set -o nounset -set -o pipefail -IFS=$'\n\t' - -# Define variables -YQ_IMAGE="mikefarah/yq" -YQ_VERSION="4.12.0" - -# Define function to run yq inside container -run_yq() { - docker run --rm -v "$(pwd):/workdir" "$YQ_IMAGE:$YQ_VERSION" "$@" -} - -# Call function with arguments -run_yq "$@" diff --git a/{{cookiecutter.project_slug}}/.osparc/metadata.yml b/{{cookiecutter.project_slug}}/.osparc/metadata.yml index 2f465269..9dbf653a 100644 --- a/{{cookiecutter.project_slug}}/.osparc/metadata.yml +++ b/{{cookiecutter.project_slug}}/.osparc/metadata.yml @@ -1,11 +1,10 @@ name: {{ cookiecutter.project_name }} key: simcore/services/{%- if cookiecutter.project_type == "computational" -%}comp{%- elif cookiecutter.project_type == "dynamic" -%}dynamic{%- endif -%}/{{ cookiecutter.project_name.lower().replace(' ', '-') }} type: {{ cookiecutter.project_type }} -integration-version: 1.0.0 +integration-version: 2.0.0 version: {{ cookiecutter.version }} -version_display: {{ cookiecutter.version_display }} description: {{ cookiecutter.project_short_description }} -contact: {{ cookiecutter.contact_email }} +contact: {{ cookiecutter.author_email }} thumbnail: https://github.com/ITISFoundation/osparc-assets/blob/cb43207b6be2f4311c93cd963538d5718b41a023/assets/default-thumbnail-cookiecutter-osparc-service.png?raw=true authors: - name: {{ cookiecutter.author_name }} diff --git a/{{cookiecutter.project_slug}}/Makefile b/{{cookiecutter.project_slug}}/Makefile index 6f2d23b5..7b41d263 100644 --- a/{{cookiecutter.project_slug}}/Makefile +++ b/{{cookiecutter.project_slug}}/Makefile @@ -9,119 +9,59 @@ export VCS_REF := $(shell git rev-parse --short HEAD 2> /dev/null || echo unv export VCS_STATUS := $(if $(shell git status -s 2> /dev/null || echo unversioned repo),'modified/untracked','clean') export BUILD_DATE := $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") -export DOCKER_REGISTRY ?= {{ cookiecutter.default_docker_registry }} export DOCKER_IMAGE_NAME ?= {{ cookiecutter.project_name.lower().replace(' ', '-') }} -export DOCKER_IMAGE_TAG ?= $(shell cat VERSION 2> /dev/null || echo undefined) - -export COMPOSE_INPUT_DIR := ./validation/input -export COMPOSE_OUTPUT_DIR := .tmp/output +export DOCKER_IMAGE_TAG ?= {{ cookiecutter.version }} OSPARC_DIR:=$(CURDIR)/.osparc APP_NAME := {{ cookiecutter.project_slug }} +# Builds new service version ---------------------------------------------------------------------------- - - -# INTEGRATION ----------------------------------------------------------------- -METADATA := .osparc/metadata.yml - - -.PHONY: VERSION -VERSION: $(METADATA) ## generates VERSION from metadata - # updating $@ from $< - @$(OSPARC_DIR)/bin/ooil get-version --metadata-file $< > $@ - -service.cli/run: $(METADATA) ## generates run from metadata - # Updates adapter script from metadata in $< - @$(OSPARC_DIR)/bin/ooil run-creator --metadata $< --runscript $@ - -docker-compose.yml: $(METADATA) ## generates docker-compose - # Injects metadata from $< as labels - @$(OSPARC_DIR)/bin/ooil compose --to-spec-file $@ --metadata $< - - - -# BUILD ----------------------------------------------------------------- - -define _docker_compose_build -export DOCKER_BUILD_TARGET=$(if $(findstring -devel,$@),development,$(if $(findstring -cache,$@),cache,production)); \ - docker compose -f docker-compose.yml build $(if $(findstring -nc,$@),--no-cache,); -endef - - -.PHONY: build build-devel build-nc build-devel-nc -build build-devel build-nc build-devel-nc: VERSION docker-compose.yml service.cli/run ## builds image - # building image local/${DOCKER_IMAGE_NAME}... - @$(call _docker_compose_build) - -define show-meta - $(foreach iid,$(shell docker images */$(1):* -q | sort | uniq),\ - docker image inspect $(iid) | jq '.[0] | .RepoTags, .ContainerConfig.Labels, .Config.Labels';) +define _bumpversion + # upgrades as $(subst $(1),,$@) version, commits and tags + @docker run -it --rm -v $(PWD):/${DOCKER_IMAGE_NAME} \ + -u $(shell id -u):$(shell id -g) \ + itisfoundation/ci-service-integration-library:v1.0.4 \ + sh -c "cd /${DOCKER_IMAGE_NAME} && bump2version --verbose --list --config-file $(1) $(subst $(2),,$@)" endef +.PHONY: version-patch version-minor version-major +version-patch version-minor version-major: .bumpversion.cfg ## increases service's version + @make compose-spec + @$(call _bumpversion,$<,version-) + @make compose-spec -.PHONY: info-build -info-build: ## displays info on the built image - # Built images - @docker images */$(DOCKER_IMAGE_NAME):* - # Tags and labels - @$(call show-meta,$(DOCKER_IMAGE_NAME)) - - -# TESTS----------------------------------------------------------------- -.PHONY: test tests -test tests: ## runs validation tests - @$(OSPARC_DIR)/bin/ooil test . - - - -# PUBLISHING ----------------------------------------------------------------- +.PHONY: compose-spec +compose-spec: ## runs ooil to assemble the docker-compose.yml file + @docker run --rm -v $(PWD):/${DOCKER_IMAGE_NAME} \ + -u $(shell id -u):$(shell id -g) \ + itisfoundation/ci-service-integration-library:v1.0.4 \ + sh -c "cd /${DOCKER_IMAGE_NAME} && ooil compose" -.PHONY: version-service-patch version-service-minor version-service-major -version-service-patch version-service-minor version-service-major: $(METADATA) ## kernel/service versioning as patch - $(OSPARC_DIR)/bin/ooil bump-version --metadata-file $< --upgrade $(subst version-service-,,$@) - # syncing metadata upstream - @$(MAKE) VERSION +build: | compose-spec ## build docker image + docker compose build - -.PHONY: tag-local -tag-local: - docker tag ${DOCKER_REGISTRY}/${DOCKER_IMAGE_NAME}:$(if $(findstring version,$@),$(DOCKER_IMAGE_NAME),latest) local/$(DOCKER_IMAGE_NAME):production - -.PHONY: push push-force push-version push-latest pull-latest pull-version tag-latest tag-version -tag-latest tag-version: - docker tag simcore/services/{%- if cookiecutter.project_type == "computational" -%}comp{%- elif cookiecutter.project_type == "dynamic" -%}dynamic{%- endif -%}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG} $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_NAME):$(if $(findstring version,$@),$(DOCKER_IMAGE_TAG),latest) - -version_valid = $(shell test $$(echo $(DOCKER_IMAGE_TAG) | cut --fields=1 --delimiter=.) -gt 0 > /dev/null && echo "image version is valid") -version_exists = $(shell DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) > /dev/null && echo "image already exists on $(DOCKER_REGISTRY)") -push push-force: ## pushes (resp. force) services to the registry if service not available in registry. - @$(if $(findstring force,$@),,\ - $(if $(call version_valid),$(info version is valid), $(error $(DOCKER_IMAGE_TAG) is not a valid version (major>=1)))\ - $(if $(call version_exists),$(error $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) already exists on $(DOCKER_REGISTRY)), $(info no version found on $(DOCKER_REGISTRY)))\ - ) - @$(MAKE) push-version; - @$(MAKE) push-latest; +# To test built service locally ------------------------------------------------------------------------- +.PHONY: run-local +run-local: ## runs image with local configuration + docker compose --file docker-compose-local.yml up .PHONY: publish-local -publish-local: ## push to local throw away registry to test integration - docker tag simcore/services/{%- if cookiecutter.project_type == "computational" -%}comp{%- elif cookiecutter.project_type == "dynamic" -%}dynamic{%- endif -%}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG} registry:5000/simcore/services/{%- if cookiecutter.project_type == "computational" -%}comp{%- elif cookiecutter.project_type == "dynamic" -%}dynamic{%- endif -%}/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) - docker push registry:5000/simcore/services/{%- if cookiecutter.project_type == "computational" -%}comp{%- elif cookiecutter.project_type == "dynamic" -%}dynamic{%- endif -%}/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) +publish-local: ## push to local oSPARC to test integration. It requires the oSPARC platform running on your computer, you can find more information here: https://github.com/ITISFoundation/osparc-simcore/blob/master/README.md + docker tag simcore/services/dynamic/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG} registry:5000/simcore/services/dynamic/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) + docker push registry:5000/simcore/services/dynamic/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) @curl registry:5000/v2/_catalog | jq - -push-latest push-version: ## publish service to registry with latest/version tag - # pushing '$(DOCKER_REGISTRY)/$(DOCKER_IMAGE_NAME):$(if $(findstring version,$@),$(DOCKER_IMAGE_TAG),latest)'... - @$(MAKE) tag-$(subst push-,,$@) - @docker push $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_NAME):$(if $(findstring version,$@),$(DOCKER_IMAGE_TAG),latest) - # pushed '$(DOCKER_REGISTRY)/$(DOCKER_IMAGE_NAME):$(if $(findstring version,$@),$(DOCKER_IMAGE_TAG),latest)' - -pull-latest pull-version: ## pull service from registry - @docker pull $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_NAME):$(if $(findstring version,$@),$(DOCKER_IMAGE_TAG),latest) - +.PHONY: help +help: ## this colorful help + @echo "Recipes for '$(notdir $(CURDIR))':" + @echo "" + @awk 'BEGIN {FS = ":.*?## "} /^[[:alpha:][:space:]_-]+:.*?## / {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) + @echo "" -# COOCKIECUTTER ----------------------------------------------------------------- +# COOKIECUTTER ----------------------------------------------------------------- .PHONY: replay replay: .cookiecutterrc ## re-applies cookiecutter @@ -132,7 +72,6 @@ replay: .cookiecutterrc ## re-applies cookiecutter "{{ cookiecutter._template }}" - .PHONY: info info: ## general info # env vars: version control @@ -140,11 +79,7 @@ info: ## general info @echo " VCS_REF : $(VCS_REF)" @echo " VCS_STATUS : $(VCS_STATUS)" # env vars: docker - @echo " DOCKER_REGISTRY : $(DOCKER_REGISTRY)" - @echo " DOCKER_IMAGE_NAME : $(DOCKER_IMAGE_NAME)" @echo " DOCKER_IMAGE_TAG : $(DOCKER_IMAGE_TAG)" - @echo " COMPOSE_INPUT_DIR : $(COMPOSE_INPUT_DIR)" - @echo " COMPOSE_OUTPUT_DIR : $(COMPOSE_OUTPUT_DIR)" @echo " BUILD_DATE : $(BUILD_DATE)" # exe: recommended dev tools @echo ' git : $(shell git --version 2>/dev/null || echo not found)' @@ -154,21 +89,10 @@ info: ## general info @echo ' python : $(shell python3 --version 2>/dev/null || echo not found )' @echo ' docker : $(shell docker --version)' @echo ' docker buildx : $(shell docker buildx version)' - @echo ' docker-compose : $(shell docker-compose --version)' - # exe: integration tools - @echo ' ooil version : $(shell $(OSPARC_DIR)/bin/ooil --version)' - - + @echo ' docker compose : $(shell docker compose --version)' # MISC ----------------------------------------------------------------- -.PHONY: help -help: ## this colorful help - @echo "Recipes for '$(notdir $(CURDIR))':" - @echo "" - @awk --posix 'BEGIN {FS = ":.*?## "} /^[[:alpha:][:space:]_-]+:.*?## / {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) - @echo "" - .PHONY: clean git_clean_args = -dxf --exclude=.vscode/ From 0f33464654cc1e561ccc6a9652e15578ed4b76c7 Mon Sep 17 00:00:00 2001 From: Javier Garcia Ordonez Date: Wed, 28 Aug 2024 18:45:56 +0200 Subject: [PATCH 04/11] new GitHub actions only (in sync w new Makefile) --- dummy | 1 + .../.github/workflows/build-test-publish.yml | 53 ------------------- 2 files changed, 1 insertion(+), 53 deletions(-) create mode 160000 dummy delete mode 100644 {{cookiecutter.project_slug}}/.github/workflows/build-test-publish.yml diff --git a/dummy b/dummy new file mode 160000 index 00000000..6c986e5b --- /dev/null +++ b/dummy @@ -0,0 +1 @@ +Subproject commit 6c986e5bce50f0f5f8ae54edc5948c5e07823586 diff --git a/{{cookiecutter.project_slug}}/.github/workflows/build-test-publish.yml b/{{cookiecutter.project_slug}}/.github/workflows/build-test-publish.yml deleted file mode 100644 index 0187e6c2..00000000 --- a/{{cookiecutter.project_slug}}/.github/workflows/build-test-publish.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Github-CI Push/PR {{ cookiecutter.project_slug }} - -on: - push: - pull_request: - -env: - # secrets can be set in settings/secrets on github - DOCKER_REGISTRY: {{ "${{ secrets.DOCKER_REGISTRY }}" }} - DOCKER_USERNAME: {{ "${{ secrets.DOCKER_USERNAME }}" }} - DOCKER_PASSWORD: {{ "${{ secrets.DOCKER_PASSWORD }}" }} - -jobs: - build: - name: building {{ cookiecutter.project_slug }} - runs-on: {{ "${{ matrix.os }}" }} - strategy: - matrix: - python: [3.9] - os: [ubuntu-22.04] - fail-fast: false - steps: - - uses: actions/checkout@v3 - - name: setup python environment - uses: actions/setup-python@v4 - with: - python-version: {{ "${{ matrix.python }}" }} - - name: show versions - run: ./.github/show_system_versions.bash - - name: set owner variable - run: echo "OWNER=${GITHUB_REPOSITORY%/*}" >> $GITHUB_ENV - - name: set docker image tag - if: github.ref != 'refs/heads/master' - run: echo "DOCKER_IMAGE_TAG=${GITHUB_REF##*/}" >> $GITHUB_ENV - - name: get current image if available - run: make pull-latest || true - - name: build - run: | - make VERSION - make build - make info-build - - name: test - run: make tests - # - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - # name: push - # run: | - # ./.github/dockerhub_login.bash - # make push - # - if: github.event_name == 'push' && github.ref != 'refs/heads/master' - # name: push - # run: | - # ./.github/dockerhub_login.bash - # make push-version From f65d4754dbaf8b6bdd52c1faea134f445c8c99ad Mon Sep 17 00:00:00 2001 From: Javier Garcia Ordonez <56032114+JavierGOrdonnez@users.noreply.github.com> Date: Fri, 7 Mar 2025 13:23:43 +0100 Subject: [PATCH 05/11] use latest ci-service-integration-library --- .../.github/workflows/check-image.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/{{cookiecutter.project_slug}}/.github/workflows/check-image.yml b/{{cookiecutter.project_slug}}/.github/workflows/check-image.yml index 0aed71b0..4a83c1cf 100644 --- a/{{cookiecutter.project_slug}}/.github/workflows/check-image.yml +++ b/{{cookiecutter.project_slug}}/.github/workflows/check-image.yml @@ -9,15 +9,15 @@ jobs: - name: Checkout repo content uses: actions/checkout@v2 - name: ooil version - uses: docker://itisfoundation/ci-service-integration-library:v1.0.4 + uses: docker://itisfoundation/ci-service-integration-library:latest with: args: ooil --version - name: Assemble docker compose spec - uses: docker://itisfoundation/ci-service-integration-library:v1.0.4 + uses: docker://itisfoundation/ci-service-integration-library:latest with: args: ooil compose - name: Build all images if multiple - uses: docker://itisfoundation/ci-service-integration-library:v1.0.4 + uses: docker://itisfoundation/ci-service-integration-library:latest with: args: docker compose build - name: test From 4059e07b56e17bf93d2ea57f409ecd2803331307 Mon Sep 17 00:00:00 2001 From: Javier Garcia Ordonez <56032114+JavierGOrdonnez@users.noreply.github.com> Date: Fri, 7 Mar 2025 13:41:18 +0100 Subject: [PATCH 06/11] Update publish local --- {{cookiecutter.project_slug}}/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/{{cookiecutter.project_slug}}/Makefile b/{{cookiecutter.project_slug}}/Makefile index 7b41d263..9e4968fb 100644 --- a/{{cookiecutter.project_slug}}/Makefile +++ b/{{cookiecutter.project_slug}}/Makefile @@ -48,9 +48,9 @@ run-local: ## runs image with local configuration docker compose --file docker-compose-local.yml up .PHONY: publish-local -publish-local: ## push to local oSPARC to test integration. It requires the oSPARC platform running on your computer, you can find more information here: https://github.com/ITISFoundation/osparc-simcore/blob/master/README.md - docker tag simcore/services/dynamic/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG} registry:5000/simcore/services/dynamic/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) - docker push registry:5000/simcore/services/dynamic/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) +publish-local: ## push to local throw away registry to test integration + docker tag simcore/services/{%- if cookiecutter.project_type == "computational" -%}comp{%- elif cookiecutter.project_type == "dynamic" -%}dynamic{%- endif -%}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG} registry:5000/simcore/services/{%- if cookiecutter.project_type == "computational" -%}comp{%- elif cookiecutter.project_type == "dynamic" -%}dynamic{%- endif -%}/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) + docker push registry:5000/simcore/services/{%- if cookiecutter.project_type == "computational" -%}comp{%- elif cookiecutter.project_type == "dynamic" -%}dynamic{%- endif -%}/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) @curl registry:5000/v2/_catalog | jq .PHONY: help From 2abae4d933d43f834e618f27fa0ec6e9add61d8f Mon Sep 17 00:00:00 2001 From: Javier Garcia Ordonez Date: Fri, 7 Mar 2025 13:55:34 +0100 Subject: [PATCH 07/11] updated few things to resemble Jupyter Pysonic (recently updated by Andrei) --- dummy | 1 - .../.github/workflows/check-image.yml | 2 -- {{cookiecutter.project_slug}}/.osparc/metadata.yml | 2 +- {{cookiecutter.project_slug}}/.osparc/runtime.yml | 4 ++++ 4 files changed, 5 insertions(+), 4 deletions(-) delete mode 160000 dummy diff --git a/dummy b/dummy deleted file mode 160000 index 6c986e5b..00000000 --- a/dummy +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6c986e5bce50f0f5f8ae54edc5948c5e07823586 diff --git a/{{cookiecutter.project_slug}}/.github/workflows/check-image.yml b/{{cookiecutter.project_slug}}/.github/workflows/check-image.yml index 4a83c1cf..128ff65e 100644 --- a/{{cookiecutter.project_slug}}/.github/workflows/check-image.yml +++ b/{{cookiecutter.project_slug}}/.github/workflows/check-image.yml @@ -20,5 +20,3 @@ jobs: uses: docker://itisfoundation/ci-service-integration-library:latest with: args: docker compose build - - name: test - run: make tests diff --git a/{{cookiecutter.project_slug}}/.osparc/metadata.yml b/{{cookiecutter.project_slug}}/.osparc/metadata.yml index 9dbf653a..186a7caa 100644 --- a/{{cookiecutter.project_slug}}/.osparc/metadata.yml +++ b/{{cookiecutter.project_slug}}/.osparc/metadata.yml @@ -1,7 +1,7 @@ name: {{ cookiecutter.project_name }} key: simcore/services/{%- if cookiecutter.project_type == "computational" -%}comp{%- elif cookiecutter.project_type == "dynamic" -%}dynamic{%- endif -%}/{{ cookiecutter.project_name.lower().replace(' ', '-') }} type: {{ cookiecutter.project_type }} -integration-version: 2.0.0 +integration-version: 1.0.0 version: {{ cookiecutter.version }} description: {{ cookiecutter.project_short_description }} contact: {{ cookiecutter.author_email }} diff --git a/{{cookiecutter.project_slug}}/.osparc/runtime.yml b/{{cookiecutter.project_slug}}/.osparc/runtime.yml index c0d3d9f3..aa98bcc7 100644 --- a/{{cookiecutter.project_slug}}/.osparc/runtime.yml +++ b/{{cookiecutter.project_slug}}/.osparc/runtime.yml @@ -1,5 +1,9 @@ restart-policy: no-restart settings: + - name: constraints + type: string + value: + - node.platform.os == linux - name: Resources type: Resources value: From 64f1a8aa9e1d43b67f66528e4ddbf8a9778d28d6 Mon Sep 17 00:00:00 2001 From: Javier Garcia Ordonez Date: Mon, 10 Mar 2025 10:25:29 +0100 Subject: [PATCH 08/11] update ci-service-integration-library in Makefile compose-spec --- {{cookiecutter.project_slug}}/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/{{cookiecutter.project_slug}}/Makefile b/{{cookiecutter.project_slug}}/Makefile index 9e4968fb..8a12d1c7 100644 --- a/{{cookiecutter.project_slug}}/Makefile +++ b/{{cookiecutter.project_slug}}/Makefile @@ -22,7 +22,7 @@ define _bumpversion # upgrades as $(subst $(1),,$@) version, commits and tags @docker run -it --rm -v $(PWD):/${DOCKER_IMAGE_NAME} \ -u $(shell id -u):$(shell id -g) \ - itisfoundation/ci-service-integration-library:v1.0.4 \ + itisfoundation/ci-service-integration-library:latest\ sh -c "cd /${DOCKER_IMAGE_NAME} && bump2version --verbose --list --config-file $(1) $(subst $(2),,$@)" endef @@ -36,7 +36,7 @@ version-patch version-minor version-major: .bumpversion.cfg ## increases service compose-spec: ## runs ooil to assemble the docker-compose.yml file @docker run --rm -v $(PWD):/${DOCKER_IMAGE_NAME} \ -u $(shell id -u):$(shell id -g) \ - itisfoundation/ci-service-integration-library:v1.0.4 \ + itisfoundation/ci-service-integration-library:latest\ sh -c "cd /${DOCKER_IMAGE_NAME} && ooil compose" build: | compose-spec ## build docker image From 7c2dd45cee47c823bf2ff523a29f30efc9ee42ec Mon Sep 17 00:00:00 2001 From: Javier Garcia Ordonez Date: Mon, 10 Mar 2025 11:27:53 +0100 Subject: [PATCH 09/11] cleaning && adding necessary new files --- .../.osparc/metadata.yml | 2 ++ .../.osparc/runtime.yml | 5 ++-- .../docker-compose-local.yml | 7 ++++++ .../service.cli/execute.sh | 24 +++++++------------ {{cookiecutter.project_slug}}/service.cli/run | 16 +++++++++++++ 5 files changed, 36 insertions(+), 18 deletions(-) create mode 100644 {{cookiecutter.project_slug}}/docker-compose-local.yml create mode 100755 {{cookiecutter.project_slug}}/service.cli/run diff --git a/{{cookiecutter.project_slug}}/.osparc/metadata.yml b/{{cookiecutter.project_slug}}/.osparc/metadata.yml index 186a7caa..3fa126d0 100644 --- a/{{cookiecutter.project_slug}}/.osparc/metadata.yml +++ b/{{cookiecutter.project_slug}}/.osparc/metadata.yml @@ -17,6 +17,7 @@ inputs: label: input_{{ i+1 }}_label description: The input {{ i+1 }} description type: string + # allowed input / output types are "string", "number", "bool", "data:*/*" defaultValue: some_value(optional) fileToKeyMap: somefilename.ext: input_{{ i+1 }} @@ -28,6 +29,7 @@ outputs: label: output_{{ i+1 }}_label description: The input {{ i+1 }} description type: string + # allowed input / output types are "string", "number", "bool", "data:*/*" fileToKeyMap: somefilename.ext: output_{{ i+1 }} {% endfor %} diff --git a/{{cookiecutter.project_slug}}/.osparc/runtime.yml b/{{cookiecutter.project_slug}}/.osparc/runtime.yml index aa98bcc7..8e589f00 100644 --- a/{{cookiecutter.project_slug}}/.osparc/runtime.yml +++ b/{{cookiecutter.project_slug}}/.osparc/runtime.yml @@ -7,7 +7,6 @@ settings: - name: Resources type: Resources value: - Limits: + Limits: # TODO Default resource limits. Change if necessary. NanoCPUs: 1000000000 # 100% of CPU cycles on 1 CPU - MemoryBytes: 2147483648 # 2 Gigabytes - + MemoryBytes: 2147483648 # 2 Gigabytes \ No newline at end of file diff --git a/{{cookiecutter.project_slug}}/docker-compose-local.yml b/{{cookiecutter.project_slug}}/docker-compose-local.yml new file mode 100644 index 00000000..151bdce7 --- /dev/null +++ b/{{cookiecutter.project_slug}}/docker-compose-local.yml @@ -0,0 +1,7 @@ +version: '3.7' +services: + {{ cookiecutter.project_slug }}: + image: simcore/services/{%- if cookiecutter.project_type == "computational" -%}comp{%- elif cookiecutter.project_type == "dynamic" -%}dynamic{%- endif -%}/{{ cookiecutter.project_name.lower().replace(' ', '-') }}:{{ cookiecutter.version }} + volumes: + - ${PWD}/validation/input:/input + - ${PWD}/validation/output:/output \ No newline at end of file diff --git a/{{cookiecutter.project_slug}}/service.cli/execute.sh b/{{cookiecutter.project_slug}}/service.cli/execute.sh index 1b6f0be2..842a0aa5 100755 --- a/{{cookiecutter.project_slug}}/service.cli/execute.sh +++ b/{{cookiecutter.project_slug}}/service.cli/execute.sh @@ -4,7 +4,7 @@ set -o errexit set -o nounset IFS=$(printf '\n\t') -cd /home/scu/{{ cookiecutter.project_package_name }} +cd /home/scu/src/{{ cookiecutter.project_package_name }} echo "starting service as" echo User : "$(id "$(whoami)")" @@ -12,25 +12,19 @@ echo Workdir : "$(pwd)" echo "..." echo # ---------------------------------------------------------------- -# This script shall be modified according to the needs in order to run the service -# The inputs defined in ${INPUT_FOLDER}/inputs.json are available as env variables by their key in capital letters -# For example: input_1 -> $INPUT_1 +# This script shall be modified according to the needs in order to run the service. You can use the inputs in ${INPUT_FOLDER} +# then retrieve the output and move it to the $OUTPUT_FOLDER as defined in the output labels +# For example: cp output.csv $OUTPUT_FOLDER or to $OUTPUT_FOLDER/outputs.json using jq +# TODO: Replace following # put the code to execute the service here # For example: env ls -al "${INPUT_FOLDER}" +# or for example, to execute a python script on some input data: +env +python3 myscript.py -i path/to/input_data/input_file.txt +cp output_file.json "${OUTPUT_FOLDER}"/outputs.json -# then retrieve the output and move it to the $OUTPUT_FOLDER -# as defined in the output labels -# For example: cp output.csv $OUTPUT_FOLDER or to $OUTPUT_FOLDER/outputs.json using jq -#TODO: Replace following -cat > "${OUTPUT_FOLDER}"/outputs.json << EOF -{ -{%- for i in range(cookiecutter.number_of_outputs|int) -%} -{%- if i > 0 -%},{%- endif %} - "output_{{ i+1 }}":"some_stuff" -{%- endfor %} -} EOF diff --git a/{{cookiecutter.project_slug}}/service.cli/run b/{{cookiecutter.project_slug}}/service.cli/run new file mode 100755 index 00000000..fd9787b1 --- /dev/null +++ b/{{cookiecutter.project_slug}}/service.cli/run @@ -0,0 +1,16 @@ + +#!/bin/sh +#--------------------------------------------------------------- +# AUTO-GENERATED CODE, do not modify this will be overwritten!!! +#--------------------------------------------------------------- +# shell strict mode: +set -o errexit +set -o nounset +IFS=$(printf '\n\t') +cd "$(dirname "$0")" + +INPUT_1=$INPUT_FOLDER/input.zip +export INPUT_1 + +exec execute.sh + \ No newline at end of file From a42a56cee533e6a8c1f812b37436aa1ae1373d94 Mon Sep 17 00:00:00 2001 From: Javier Garcia Ordonez <56032114+JavierGOrdonnez@users.noreply.github.com> Date: Thu, 31 Jul 2025 13:29:05 +0200 Subject: [PATCH 10/11] pin CI service integration library version Co-authored-by: Andrei Neagu <5694077+GitHK@users.noreply.github.com> --- {{cookiecutter.project_slug}}/.github/workflows/check-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.project_slug}}/.github/workflows/check-image.yml b/{{cookiecutter.project_slug}}/.github/workflows/check-image.yml index 128ff65e..029ee47e 100644 --- a/{{cookiecutter.project_slug}}/.github/workflows/check-image.yml +++ b/{{cookiecutter.project_slug}}/.github/workflows/check-image.yml @@ -9,7 +9,7 @@ jobs: - name: Checkout repo content uses: actions/checkout@v2 - name: ooil version - uses: docker://itisfoundation/ci-service-integration-library:latest + uses: docker://itisfoundation/ci-service-integration-library:v2.1.25 with: args: ooil --version - name: Assemble docker compose spec From 4c8823e54909861a2f5725f93830d79c7588f0bc Mon Sep 17 00:00:00 2001 From: Javier Garcia Ordonez <56032114+JavierGOrdonnez@users.noreply.github.com> Date: Thu, 31 Jul 2025 13:29:44 +0200 Subject: [PATCH 11/11] pin CI service integration library version Co-authored-by: Andrei Neagu <5694077+GitHK@users.noreply.github.com> --- .../.github/workflows/check-image.yml | 4 ++-- {{cookiecutter.project_slug}}/Makefile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/{{cookiecutter.project_slug}}/.github/workflows/check-image.yml b/{{cookiecutter.project_slug}}/.github/workflows/check-image.yml index 029ee47e..f70a5ff1 100644 --- a/{{cookiecutter.project_slug}}/.github/workflows/check-image.yml +++ b/{{cookiecutter.project_slug}}/.github/workflows/check-image.yml @@ -13,10 +13,10 @@ jobs: with: args: ooil --version - name: Assemble docker compose spec - uses: docker://itisfoundation/ci-service-integration-library:latest + uses: docker://itisfoundation/ci-service-integration-library:v2.1.25 with: args: ooil compose - name: Build all images if multiple - uses: docker://itisfoundation/ci-service-integration-library:latest + uses: docker://itisfoundation/ci-service-integration-library:v2.1.25 with: args: docker compose build diff --git a/{{cookiecutter.project_slug}}/Makefile b/{{cookiecutter.project_slug}}/Makefile index 8a12d1c7..e696bf61 100644 --- a/{{cookiecutter.project_slug}}/Makefile +++ b/{{cookiecutter.project_slug}}/Makefile @@ -22,7 +22,7 @@ define _bumpversion # upgrades as $(subst $(1),,$@) version, commits and tags @docker run -it --rm -v $(PWD):/${DOCKER_IMAGE_NAME} \ -u $(shell id -u):$(shell id -g) \ - itisfoundation/ci-service-integration-library:latest\ + itisfoundation/ci-service-integration-library:v2.1.25\ sh -c "cd /${DOCKER_IMAGE_NAME} && bump2version --verbose --list --config-file $(1) $(subst $(2),,$@)" endef @@ -36,7 +36,7 @@ version-patch version-minor version-major: .bumpversion.cfg ## increases service compose-spec: ## runs ooil to assemble the docker-compose.yml file @docker run --rm -v $(PWD):/${DOCKER_IMAGE_NAME} \ -u $(shell id -u):$(shell id -g) \ - itisfoundation/ci-service-integration-library:latest\ + itisfoundation/ci-service-integration-library:v2.1.25\ sh -c "cd /${DOCKER_IMAGE_NAME} && ooil compose" build: | compose-spec ## build docker image