Merge pull request #100 from AbsaOSS/feature/83-components-discovery

Feature/83 components discovery

api/src/main/scala/za/co/absa/hyperdrive/ingestor/api/ComponentFactory.scala

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.hyperdrive.ingestor.api
+
+import org.apache.commons.configuration2.Configuration
+
+trait ComponentFactory[T] extends HasComponentAttributes {
+  def apply(config: Configuration): T
+}

api/src/main/scala/za/co/absa/hyperdrive/ingestor/api/ComponentFactoryProvider.scala

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.hyperdrive.ingestor.api
+
+/**
+ * Implementations of this trait are responsible for providing an instance of a [[ComponentFactory]] via the
+ * service provider interface (SPI) and need to be registered in a provider configuration file under META-INF/services
+ * Implementations must have a no-args constructor to be instantiable.
+ */
+trait ComponentFactoryProvider[T <: ComponentFactory[_]] {
+  def getComponentFactory: T
+}

api/src/main/scala/za/co/absa/hyperdrive/ingestor/api/HasComponentAttributes.scala

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.hyperdrive.ingestor.api
+
+case class PropertyMetadata(
+                             /**
+                              * A human readable label for this property.
+                              */
+                             label: String,
+
+                             /**
+                              * An optional text that gives a detailed description of this property.
+                              */
+                            hint: Option[String],
+
+                             /**
+                              * true if the property is required, false otherwise
+                              */
+                            required: Boolean)
+
+/**
+ * This trait should be implemented by all implementations of [[ComponentFactory]] to provide a self-description
+ * how the component should be used and configured. This information may be used by external applications
+ * that automatically discover components
+ */
+trait HasComponentAttributes {
+  /**
+   * @return a human readable name of the component.
+   */
+  def getName: String
+
+  /**
+   * @return a description for the component.
+   */
+  def getDescription: String
+
+  /**
+   * @return a map describing configuration properties for this component. The keys have to be unique to avoid
+   *         name clashes with properties from other components.
+   */
+  def getProperties: Map[String, PropertyMetadata]
+
+  /**
+   * @return a prefix to be used for arbitrary extra configuration. Typically extra configuration is required
+   *         to pass on configuration properties, e.g. to DataStreamWriter.options
+   */
+  def getExtraConfigurationPrefix: Option[String] = None
+}

api/src/main/scala/za/co/absa/hyperdrive/ingestor/api/decoder/StreamDecoderFactory.scala

@@ -15,8 +15,6 @@
 
 package za.co.absa.hyperdrive.ingestor.api.decoder
 
-import org.apache.commons.configuration2.Configuration
+import za.co.absa.hyperdrive.ingestor.api.ComponentFactory
 
-trait StreamDecoderFactory {
-  def apply(config: Configuration): StreamDecoder
-}
+trait StreamDecoderFactory extends ComponentFactory[StreamDecoder]

api/src/main/scala/za/co/absa/hyperdrive/ingestor/api/decoder/StreamDecoderFactoryProvider.scala

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.hyperdrive.ingestor.api.decoder
+
+import za.co.absa.hyperdrive.ingestor.api.ComponentFactoryProvider
+
+trait StreamDecoderFactoryProvider extends ComponentFactoryProvider[StreamDecoderFactory]

api/src/main/scala/za/co/absa/hyperdrive/ingestor/api/manager/StreamManagerFactory.scala

@@ -15,8 +15,6 @@
 
 package za.co.absa.hyperdrive.ingestor.api.manager
 
-import org.apache.commons.configuration2.Configuration
+import za.co.absa.hyperdrive.ingestor.api.ComponentFactory
 
-trait StreamManagerFactory {
-  def apply(config: Configuration): StreamManager
-}
+trait StreamManagerFactory extends ComponentFactory[StreamManager]

api/src/main/scala/za/co/absa/hyperdrive/ingestor/api/manager/StreamManagerFactoryProvider.scala

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.hyperdrive.ingestor.api.manager
+
+import za.co.absa.hyperdrive.ingestor.api.ComponentFactoryProvider
+
+trait StreamManagerFactoryProvider extends ComponentFactoryProvider[StreamManagerFactory]

api/src/main/scala/za/co/absa/hyperdrive/ingestor/api/reader/StreamReaderFactory.scala

@@ -15,8 +15,6 @@
 
 package za.co.absa.hyperdrive.ingestor.api.reader
 
-import org.apache.commons.configuration2.Configuration
+import za.co.absa.hyperdrive.ingestor.api.ComponentFactory
 
-trait StreamReaderFactory {
-  def apply(conf: Configuration): StreamReader
-}
+trait StreamReaderFactory extends ComponentFactory[StreamReader]

api/src/main/scala/za/co/absa/hyperdrive/ingestor/api/reader/StreamReaderFactoryProvider.scala

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.hyperdrive.ingestor.api.reader
+
+import za.co.absa.hyperdrive.ingestor.api.ComponentFactoryProvider
+
+trait StreamReaderFactoryProvider extends ComponentFactoryProvider[StreamReaderFactory]

api/src/main/scala/za/co/absa/hyperdrive/ingestor/api/transformer/StreamTransformerFactory.scala

@@ -15,8 +15,6 @@
 
 package za.co.absa.hyperdrive.ingestor.api.transformer
 
-import org.apache.commons.configuration2.Configuration
+import za.co.absa.hyperdrive.ingestor.api.ComponentFactory
 
-trait StreamTransformerFactory {
-  def apply(config: Configuration): StreamTransformer
-}
+trait StreamTransformerFactory extends ComponentFactory[StreamTransformer]

api/src/main/scala/za/co/absa/hyperdrive/ingestor/api/transformer/StreamTransformerFactoryProvider.scala

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.hyperdrive.ingestor.api.transformer
+
+import za.co.absa.hyperdrive.ingestor.api.ComponentFactoryProvider
+
+trait StreamTransformerFactoryProvider extends ComponentFactoryProvider[StreamTransformerFactory]

api/src/main/scala/za/co/absa/hyperdrive/ingestor/api/writer/StreamWriterFactory.scala

@@ -15,8 +15,6 @@
 
 package za.co.absa.hyperdrive.ingestor.api.writer
 
-import org.apache.commons.configuration2.Configuration
+import za.co.absa.hyperdrive.ingestor.api.ComponentFactory
 
-trait StreamWriterFactory {
-  def apply(config: Configuration): StreamWriter
-}
+trait StreamWriterFactory extends ComponentFactory[StreamWriter]

api/src/main/scala/za/co/absa/hyperdrive/ingestor/api/writer/StreamWriterFactoryProvider.scala

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.hyperdrive.ingestor.api.writer
+
+import za.co.absa.hyperdrive.ingestor.api.ComponentFactoryProvider
+
+trait StreamWriterFactoryProvider extends ComponentFactoryProvider[StreamWriterFactory]

component-archetype/README.md

@@ -16,17 +16,16 @@
   ~
   -->
 
-Hyperdrive Component Archetype
-==============================
+# Hyperdrive Component Archetype
 
 This is a Maven archetype for creating custom Hyperdrive components.
 
-Creating a custom Hyperdrive components project
------------------------------------------------
+## Creating a custom Hyperdrive components project
+
 
 Download the artifact to your local maven repository
 ```
-mvn dependency:get -Dartifact=za.co.absa.hyperdrive:component-archetype:2.0.0 
+mvn dependency:get -Dartifact=za.co.absa.hyperdrive:component-archetype:3.0.0 
 ```
 
 Update the local archetype catalog
@@ -39,29 +38,38 @@ Generate a skeleton project by executing the following command
 mvn archetype:generate \
     -DarchetypeGroupId=za.co.absa.hyperdrive \
     -DarchetypeArtifactId=component-archetype \
-    -DarchetypeVersion=2.0.0 \
+    -DarchetypeVersion=3.0.0 \
     -DgroupId=<groupId> \
     -DartifactId=<artifactId> \
     -Dversion=<artifact-version> 
 ```
 - `<groupId>` is your group id, e.g. com.acme,
 - `<artifactId>` is the name for your artifact, e.g. mytransformer,
-- `<artifact-version>` is the version number of the artifact, e.g. 0.1.0-SNAPSHOT and
+- `<artifact-version>` is the version number of the artifact, e.g. 0.1.0-SNAPSHOT
+
+## Implementing a Hyperdrive component
 
-Implementing a Hyperdrive component
------------------------------------
+There are five types of Hyperdrive components: Reader, Decoder, Transformer, Writer and Manager.
 
-There are five types of Hyperdrive components: Decoder, Manager, Reader, Transformer and Writer.
+### Component stubs
 
 This archetype provides stubs for all types, which are located under `<project-root>/src/main/scala/<groupId>/`
 
-For example, if you need to implement a custom transformer, you should modify the stubs `<project-root>/src/main/scala/{groupId}/transformer/mycomponent/MyStreamTransformerImpl`
+For example, if you need to implement a custom transformer, you should modify the stubs in
+ `<project-root>/src/main/scala/{groupId}/transformer/mycomponent/MyStreamTransformerImpl`
+
+If you don't need to implement a component type, you should delete the corresponding stubs. 
+ E.g. if you don't need to implement the writer, delete the folder `<project-root>/src/main/scala/{groupId}/writer`
 
-If you don't need to implement a component type, you should delete the corresponding stubs. E.g. if you don't need to implement the writer, delete the folder `<project-root>/src/main/scala/{groupId}/writer`
+### Service provider configuration
 
+Components need to be registered using the Java Service Provider Interface (SPI). There are configuration file templates
+ under `<project-root>/src/resources/META-INF/services`. If you don't need a component type, 
+ you should delete the corresponding configuration files.
+
+A model test to verify the configuration is available under `<project-root>/test/scala/ServiceProviderConfigurationTest.scala`
 
-Building Hyperdrive components
-------------------------------
+## Building Hyperdrive components
 ```
 % cd <target-directory>/<groupId>
 % mvn clean package

component-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml

@@ -15,25 +15,31 @@
   -->
 
 <archetype-descriptor
-        xsi:schemaLocation="http://maven.apache.org/plugins/maven-archetype-plugin/archetype-descriptor/1.0.0
+    xsi:schemaLocation="http://maven.apache.org/plugins/maven-archetype-plugin/archetype-descriptor/1.0.0
         http://maven.apache.org/xsd/archetype-descriptor-1.0.0.xsd"
-        xmlns="http://maven.apache.org/plugins/maven-archetype-plugin/archetype-descriptor/1.0.0"
-        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        name="archetype">
+    xmlns="http://maven.apache.org/plugins/maven-archetype-plugin/archetype-descriptor/1.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    name="archetype">
     <fileSets>
         <fileSet filtered="false" packaged="false">
             <directory></directory>
             <includes>
-                    <include>.gitignore</include>
+                <include>.gitignore</include>
             </includes>
         </fileSet>
         <fileSet filtered="true" packaged="true">
             <directory>src/main/scala</directory>
             <includes>
                 <include>**/*.scala</include>
             </includes>
-       </fileSet>
-       <fileSet filtered="true" packaged="true">
+        </fileSet>
+        <fileSet filtered="true">
+            <directory>src/main/resources</directory>
+            <includes>
+                <include>**/*</include>
+            </includes>
+        </fileSet>
+        <fileSet filtered="true" packaged="true">
             <directory>src/test/scala</directory>
             <includes>
                 <include>**/*.scala</include>