Make component examples an optional Elyra feature (elyra-ai#2286)

Moves KFP and Airflow component examples out of Elyra core and 
into separate and optional python packages  

Co-authored-by: Kiersten Stokes <[email protected]>
Co-authored-by: Alan Chin <[email protected]>

Makefile

@@ -14,12 +14,12 @@
 # limitations under the License.
 #
 
-.PHONY: help purge install uninstall clean test-dependencies lint-server lint-ui lint yarn-install eslint-ui eslint-check-ui prettier-ui prettier-check-ui flake lint-server-dependencies dev-link dev-unlink
+.PHONY: help purge install-all install-examples install uninstall clean test-dependencies lint-server lint-ui lint yarn-install eslint-ui eslint-check-ui prettier-ui prettier-check-ui flake lint-server-dependencies dev-link dev-unlink
 .PHONY: build-ui build-server install-server watch install-extensions build-jupyterlab install-server-package check-install only-install-server
 .PHONY: test-server test-ui test-integration test-integration-debug test docs-dependencies docs dist-ui release pytest
 .PHONY: validate-runtime-images elyra-image publish-elyra-image kf-notebook-image
 .PHONY: publish-kf-notebook-image container-images publish-container-images
-
+.PHONY: build-dependencies
 SHELL:=/bin/bash
 
 TAG:=dev
@@ -70,14 +70,22 @@ uninstall:
 	- jupyter labextension uninstall @jupyter-server/resource-usage
 	pip uninstall -y elyra
 	- jupyter lab clean
+	# remove Kubeflow Pipelines example components
+	- pip uninstall -y elyra-examples-kfp-catalog
+	# remove Apache Airflow example components
+	- pip uninstall -y elyra-examples-airflow-catalog
 
 clean: purge uninstall ## Make a clean source tree and uninstall extensions
 
 test-dependencies:
 	python -m pip install --upgrade pip
 	@pip install -q -r test_requirements.txt
 
-lint-server: test-dependencies
+build-dependencies:
+	python -m pip install --upgrade pip
+	@pip install -q -r build_requirements.txt	
+
+lint-server: build-dependencies
 	flake8 elyra
 
 prettier-check-ui:
@@ -136,10 +144,21 @@ only-install-server: prepare-server build-server install-server-package
 install-server: lint-server only-install-server ## Build and install backend only
 
 install-server-package:
-	pip install --upgrade --upgrade-strategy $(UPGRADE_STRATEGY) --use-deprecated=legacy-resolver "$(shell find dist -name "elyra-*-py3-none-any.whl")[all]"
+	pip install --upgrade --upgrade-strategy $(UPGRADE_STRATEGY) --use-deprecated=legacy-resolver "$(shell find dist -name "elyra-*-py3-none-any.whl")[kfp-tekton]"
 
 install: install-server install-ui check-install ## Build and install
 
+install-all: install install-examples ## Build and install, including examples
+
+install-examples: ## Install example pipeline components 
+	# install Kubeflow Pipelines example components
+	# -> https://github.com/elyra-ai/examples/tree/master/component-catalog-connectors/kfp-example-components-connector
+	- pip install --upgrade elyra-examples-kfp-catalog
+	# install Apache Airflow example components
+	# -> https://github.com/elyra-ai/examples/tree/master/component-catalog-connectors/airflow-example-components-connector
+	- pip install --upgrade elyra-examples-airflow-catalog
+
+
 check-install:
 	jupyter serverextension list
 	jupyter server extension list

build_requirements.txt

@@ -0,0 +1,2 @@
+flake8>=3.5.0,<3.9.0
+flake8-import-order>=0.18.1

conftest.py

@@ -15,5 +15,47 @@
 #
 import pytest
 
+from elyra.metadata.metadata import Metadata
+from elyra.metadata.schemaspaces import ComponentCatalogs
+from elyra.metadata.manager import MetadataManager
+
 pytest_plugins = ["jupyter_server.pytest_plugin"]
 
+KFP_COMPONENT_CACHE_INSTANCE = {
+    "display_name": "KFP Example Components",
+    "metadata": {
+        "runtime_type": "KUBEFLOW_PIPELINES",
+        "categories": ["examples"]
+    },
+    "schema_name": "elyra-kfp-examples-catalog"
+}
+
+AIRFLOW_COMPONENT_CACHE_INSTANCE = {
+    "display_name": "Airflow Example Components",
+    "metadata": {
+        "runtime_type": "APACHE_AIRFLOW",
+        "categories": ["examples"]
+    },
+    "schema_name": "elyra-airflow-examples-catalog"
+}
+
+@pytest.fixture
+def component_cache_instance(request):
+    """Creates an instance of a component cache and removes after test."""
+    instance_name = "component_cache"
+    md_mgr = MetadataManager(schemaspace=ComponentCatalogs.COMPONENT_CATALOGS_SCHEMASPACE_ID)
+    # clean possible orphaned instance...
+    try:
+        md_mgr.remove(instance_name)
+    except Exception:
+        pass
+
+    # Attempt to create the instance
+    try:
+        component_cache_instance = md_mgr.create(instance_name, Metadata(**request.param))
+        yield component_cache_instance.name
+        md_mgr.remove(component_cache_instance.name)
+
+    # Test was not parametrized, so component instance is not needed
+    except AttributeError:
+        yield None

create-release.py

@@ -154,8 +154,8 @@ def update_version_to_release() -> None:
             f"{new_version}")
 
         sed(_source('etc/docker/kubeflow/Dockerfile'),
-            r"elyra\[all\]==.*",
-            f"elyra\[all\]=={new_version}")
+            r"elyra\[kfp-tekton,kfp-examples\]==.*",
+            f"elyra\[kfp-tekton,kfp-examples\]=={new_version}")
         sed(_source('etc/docker/elyra/Dockerfile'),
             r"    cd /tmp/elyra && make UPGRADE_STRATEGY=eager install && rm -rf /tmp/elyra",
             f"    cd /tmp/elyra \&\& git checkout tags/v{new_version} -b v{new_version} \&\& make UPGRADE_STRATEGY=eager install \&\& rm -rf /tmp/elyra")