Skip to content

Commit

Permalink
Merge pull request #890 from IBM/remote-image-fix
Browse files Browse the repository at this point in the history 
Remove DOCKER_REMOTE_IMAGE from .make.defaults
  • Loading branch information
@touma-I
touma-I authored Jan 15, 2025
2 parents 03fc0c4 + 25911e7 commit 8f0a723
Show file tree
Hide file tree
Showing 14 changed files with 14 additions and 45 deletions.
1 change: 0 additions & 1 deletion .make.defaults
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ DOCKER_REGISTRY_USER?=$(DPK_DOCKER_REGISTRY_USER)
DOCKER_REGISTRY_KEY?=$(DPK_DOCKER_REGISTRY_KEY)
DOCKER_REGISTRY_ENDPOINT?=$(DOCKER_HOSTNAME)/$(DOCKER_NAMESPACE)
DOCKER_LOCAL_IMAGE=$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION)
DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION)
DOCKER_SPARK_BASE_IMAGE_NAME=data-prep-kit-spark-$(SPARK_VERSION)
DOCKER_SPARK_BASE_IMAGE=$(DOCKER_SPARK_BASE_IMAGE_NAME):$(DOCKER_IMAGE_VERSION)
RAY_BASE_IMAGE?=docker.io/rayproject/ray:${RAY}-py310
Expand Down
3 changes: 3 additions & 0 deletions kfp/kfp_ray_components/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ else
DOCKER_IMAGE_NAME=kfp-data-processing
endif

## removed from .make.defaults and set here before loading in kind
DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION)

.PHONY: .lib-src-image
.lib-src-image:: .default.build-lib-wheel
@$(eval LIB_WHEEL_FILE := $(shell find data-processing-dist/*.whl))
Expand Down
3 changes: 3 additions & 0 deletions transforms/.make.cicd.targets
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ test-image:: .default.build-lib-wheel
$(MAKE) DOCKER_FILE=Dockerfile.python \
TRANSFORM_RUNTIME_SRC_FILE=$(TRANSFORM_PYTHON_SRC) \
DOCKER_IMAGE_NAME=$(TRANSFORM_NAME)-python \
DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) \
test-image-sequence ; \
fi ;\
fi
Expand All @@ -75,6 +76,7 @@ test-image:: .default.build-lib-wheel
TRANSFORM_RUNTIME_SRC_FILE=$(TRANSFORM_RAY_SRC) \
DOCKER_IMAGE_NAME=$(TRANSFORM_NAME)-ray \
BASE_IMAGE=$(RAY_BASE_IMAGE) \
DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) \
test-image-sequence ; \
fi ;\
fi
Expand All @@ -84,6 +86,7 @@ test-image:: .default.build-lib-wheel
TRANSFORM_RUNTIME_SRC_FILE=$(TRANSFORM_SPARK_SRC) \
DOCKER_IMAGE_NAME=$(TRANSFORM_NAME)-spark \
BASE_IMAGE=$(SPARK_BASE_IMAGE) \
DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) \
test-image-sequence ; \
fi ;\
fi
Expand Down
4 changes: 4 additions & 0 deletions transforms/.make.transforms
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ DOCKER_IMAGE_NAME?=$(TRANSFORM_NAME)-$(TRANSFORM_RUNTIME)
TRANSFORM_RUNTIME_SRC_FILE?=$(TRANSFORM_NAME)_transform_$(TRANSFORM_RUNTIME).py
TRANSFORM_TEST_FILE?=test/test_$(TRANSFORM_NAME).py

## Used when publish or building the image by legacy tansforms.
## Will be removed when all transforms get refactored
DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION)

# The following taken from https://stackoverflow.com/a/65243296/45375
# Lists all targets and optional help text found in the target.
# :: rule means we first run the help from the main make.defaults.
Expand Down
3 changes: 1 addition & 2 deletions transforms/language/doc_chunk/kfp_ray/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,7 @@ workflow-test: workflow-build
TRANSFORM_RUNTIME=$(TRANSFORM_RUNTIME) \
TRANSFORM_NAME=$(TRANSFORM_NAME) \
BUILD_SPECIFIC_RUNTIME=$(BUILD_SPECIFIC_RUNTIME) \
DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) \
PIPELINE_FILE=doc_chunk_wf.yaml .workflows.test-pipeline
PIPELINE_FILE=$(TRANSFORM_NAME)_wf.yaml .workflows.test-pipeline

.PHONY: workflow-upload
workflow-upload: workflow-build
Expand Down
1 change: 0 additions & 1 deletion transforms/language/doc_quality/kfp_ray/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ workflow-test: workflow-build
TRANSFORM_RUNTIME=$(TRANSFORM_RUNTIME) \
TRANSFORM_NAME=$(TRANSFORM_NAME) \
BUILD_SPECIFIC_RUNTIME=$(BUILD_SPECIFIC_RUNTIME) \
DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) \
PIPELINE_FILE=$(TRANSFORM_NAME)_wf.yaml .workflows.test-pipeline

.PHONY: workflow-upload
Expand Down
18 changes: 1 addition & 17 deletions transforms/language/html2parquet/kfp_ray/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,21 +27,6 @@ clean:
@# Help: Clean up the virtual environment.
rm -rf ${REPOROOT}/transforms/venv

venv::

build::

test::

test-src::

test-image::

publish::

image::

load-image::

.PHONY: workflow-build
workflow-build: workflow-venv
Expand All @@ -53,8 +38,7 @@ workflow-test: workflow-build
TRANSFORM_RUNTIME=$(TRANSFORM_RUNTIME) \
TRANSFORM_NAME=$(TRANSFORM_NAME) \
BUILD_SPECIFIC_RUNTIME=$(BUILD_SPECIFIC_RUNTIME) \
DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) \
PIPELINE_FILE=html2parquet_wf.yaml .workflows.test-pipeline
PIPELINE_FILE=$(TRANSFORM_NAME)_wf.yaml .workflows.test-pipeline

.PHONY: workflow-upload
workflow-upload:
Expand Down
1 change: 0 additions & 1 deletion transforms/language/lang_id/kfp_ray/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ workflow-test: workflow-build
TRANSFORM_RUNTIME=$(TRANSFORM_RUNTIME) \
TRANSFORM_NAME=$(TRANSFORM_NAME) \
BUILD_SPECIFIC_RUNTIME=$(BUILD_SPECIFIC_RUNTIME) \
DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) \
PIPELINE_FILE=$(TRANSFORM_NAME)_wf.yaml .workflows.test-pipeline

.PHONY: workflow-upload
Expand Down
3 changes: 1 addition & 2 deletions transforms/language/pdf2parquet/kfp_ray/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,7 @@ workflow-test: workflow-build
TRANSFORM_RUNTIME=$(TRANSFORM_RUNTIME) \
TRANSFORM_NAME=$(TRANSFORM_NAME) \
BUILD_SPECIFIC_RUNTIME=$(BUILD_SPECIFIC_RUNTIME) \
DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) \
PIPELINE_FILE=pdf2parquet_wf.yaml .workflows.test-pipeline
PIPELINE_FILE=$(TRANSFORM_NAME)_wf.yaml .workflows.test-pipeline

.PHONY: workflow-upload
workflow-upload: workflow-build
Expand Down
3 changes: 1 addition & 2 deletions transforms/language/text_encoder/kfp_ray/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,7 @@ workflow-test: workflow-build
TRANSFORM_RUNTIME=$(TRANSFORM_RUNTIME) \
TRANSFORM_NAME=$(TRANSFORM_NAME) \
BUILD_SPECIFIC_RUNTIME=$(BUILD_SPECIFIC_RUNTIME) \
DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) \
PIPELINE_FILE=text_encoder_wf.yaml .workflows.test-pipeline
PIPELINE_FILE=$(TRANSFORM_NAME)_wf.yaml .workflows.test-pipeline

.PHONY: workflow-upload
workflow-upload: workflow-build
Expand Down
16 changes: 0 additions & 16 deletions transforms/universal/doc_id/kfp_ray/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,21 +27,6 @@ clean:
@# Help: Clean up the virtual environment.
rm -rf ${REPOROOT}/transforms/venv

venv::

build::

test::

test-src::

test-image::

publish::

image::

load-image::

.PHONY: workflow-build
workflow-build: workflow-venv
Expand All @@ -53,7 +38,6 @@ workflow-test: workflow-build
TRANSFORM_RUNTIME=$(TRANSFORM_RUNTIME) \
TRANSFORM_NAME=$(TRANSFORM_NAME) \
BUILD_SPECIFIC_RUNTIME=$(BUILD_SPECIFIC_RUNTIME) \
DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) \
PIPELINE_FILE=$(TRANSFORM_NAME)_wf.yaml .workflows.test-pipeline

.PHONY: workflow-upload
Expand Down
1 change: 0 additions & 1 deletion transforms/universal/ededup/kfp_ray/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ workflow-test: workflow-build
TRANSFORM_RUNTIME=$(TRANSFORM_RUNTIME) \
TRANSFORM_NAME=$(TRANSFORM_NAME) \
BUILD_SPECIFIC_RUNTIME=$(BUILD_SPECIFIC_RUNTIME) \
DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) \
PIPELINE_FILE=$(TRANSFORM_NAME)_wf.yaml .workflows.test-pipeline

.PHONY: workflow-upload
Expand Down
1 change: 0 additions & 1 deletion transforms/universal/fdedup/kfp_ray/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ workflow-test: workflow-build
TRANSFORM_RUNTIME=$(TRANSFORM_RUNTIME) \
TRANSFORM_NAME=$(TRANSFORM_NAME) \
BUILD_SPECIFIC_RUNTIME=$(BUILD_SPECIFIC_RUNTIME) \
DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) \
PIPELINE_FILE=$(TRANSFORM_NAME)_wf.yaml .workflows.test-pipeline

.PHONY: workflow-upload
Expand Down
1 change: 0 additions & 1 deletion transforms/universal/hap/kfp_ray/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ workflow-test: workflow-build
TRANSFORM_RUNTIME=$(TRANSFORM_RUNTIME) \
TRANSFORM_NAME=$(TRANSFORM_NAME) \
BUILD_SPECIFIC_RUNTIME=$(BUILD_SPECIFIC_RUNTIME) \
DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) \
PIPELINE_FILE=$(TRANSFORM_NAME)_wf.yaml .workflows.test-pipeline

.PHONY: workflow-upload
Expand Down

0 comments on commit 8f0a723

Please sign in to comment.