From fd6e1495c2681a02e12d933cceda3c7f2388d3df Mon Sep 17 00:00:00 2001 From: jist <95856749+george0st@users.noreply.github.com> Date: Sat, 30 Mar 2024 12:52:33 +0100 Subject: [PATCH 1/6] Update kafka doc --- docs/kafka.md | 6 +++++- qgate-sln-mlrun-private.env | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/kafka.md b/docs/kafka.md index dc709bb2..3245fbba 100644 --- a/docs/kafka.md +++ b/docs/kafka.md @@ -17,7 +17,11 @@ - interactive access to the container - `docker exec -it kafka1 /bin/bash` - get kafka version `kafka-topics --version` - + - list kafka topics + - `docker exec -t kafka1 /usr/bin/kafka-topics --bootstrap-server localhost:9092 --list` + - get kafka content of topic 'aa' + - `kafka-console-consumer --bootstrap-server localhost:9092 --topic aa --from-beginning` + - or `docker exec -t kafka1 /usr/bin/kafka-console-consumer --bootstrap-server localhost:9092 --topic aa --from-beginning` ## 3. Use Kafka for tests - Update `qgate-sln-mlrun.env`, change setting for `QGATE_KAFKA` diff --git a/qgate-sln-mlrun-private.env b/qgate-sln-mlrun-private.env index 8396dbd7..94565f4a 100644 --- a/qgate-sln-mlrun-private.env +++ b/qgate-sln-mlrun-private.env @@ -31,7 +31,7 @@ QGATE_DEFINITION = ../qgate-model QGATE_DATASET = 01-size-100 # List of projects for testing e.g. agate-1, agate-2, etc. Default is empty list (all projects will be tested) -QGATE_FILTER_PROJECTS = agate-2 +QGATE_FILTER_PROJECTS = agate-1 # Path to the output directory (as off-line storage, valid for target 'parquet' and 'csv') # sample value e.g. ./output From f5458980049b1c6c18139c311ffccd5348a59559 Mon Sep 17 00:00:00 2001 From: jist <95856749+george0st@users.noreply.github.com> Date: Sat, 30 Mar 2024 12:59:35 +0100 Subject: [PATCH 2/6] Update kafka.md --- docs/kafka.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/kafka.md b/docs/kafka.md index 3245fbba..fdc271e0 100644 --- a/docs/kafka.md +++ b/docs/kafka.md @@ -19,9 +19,18 @@ - get kafka version `kafka-topics --version` - list kafka topics - `docker exec -t kafka1 /usr/bin/kafka-topics --bootstrap-server localhost:9092 --list` - - get kafka content of topic 'aa' + + - create (define) new kafka topic 'aa' + - `kafka-topics --bootstrap-server localhost:9092 --topic aa --create --partitions 3 --replication-factor 1` + - or `docker exec -t kafka1 /usr/bin/kafka-topics --bootstrap-server localhost:9092 --topic aa --create --partitions 3 --replication-factor 1` + - fire (produce) new kafka topic 'aa' + - `kafka-console-producer --bootstrap-server localhost:9092 --topic aa` + - or `docker exec -t kafka1 /usr/bin/kafka-console-producer --bootstrap-server localhost:9092 --topic aa` + - get kafka topics 'aa' - `kafka-console-consumer --bootstrap-server localhost:9092 --topic aa --from-beginning` - or `docker exec -t kafka1 /usr/bin/kafka-console-consumer --bootstrap-server localhost:9092 --topic aa --from-beginning` + + ## 3. Use Kafka for tests - Update `qgate-sln-mlrun.env`, change setting for `QGATE_KAFKA` From e8d86e325206dbe413bd76284be879aeb97c9395 Mon Sep 17 00:00:00 2001 From: jist <95856749+george0st@users.noreply.github.com> Date: Sat, 30 Mar 2024 13:00:42 +0100 Subject: [PATCH 3/6] Update kafka.md --- docs/kafka.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/kafka.md b/docs/kafka.md index fdc271e0..a11777e9 100644 --- a/docs/kafka.md +++ b/docs/kafka.md @@ -18,15 +18,15 @@ - `docker exec -it kafka1 /bin/bash` - get kafka version `kafka-topics --version` - list kafka topics - - `docker exec -t kafka1 /usr/bin/kafka-topics --bootstrap-server localhost:9092 --list` - + - `kafka-topics --bootstrap-server localhost:9092 --list` + - or `docker exec -t kafka1 /usr/bin/kafka-topics --bootstrap-server localhost:9092 --list` - create (define) new kafka topic 'aa' - `kafka-topics --bootstrap-server localhost:9092 --topic aa --create --partitions 3 --replication-factor 1` - or `docker exec -t kafka1 /usr/bin/kafka-topics --bootstrap-server localhost:9092 --topic aa --create --partitions 3 --replication-factor 1` - - fire (produce) new kafka topic 'aa' + - fire (produce) new kafka topic 'aa' (CTRL+C for finish new topic) - `kafka-console-producer --bootstrap-server localhost:9092 --topic aa` - or `docker exec -t kafka1 /usr/bin/kafka-console-producer --bootstrap-server localhost:9092 --topic aa` - - get kafka topics 'aa' + - get (consume) kafka topics 'aa' - `kafka-console-consumer --bootstrap-server localhost:9092 --topic aa --from-beginning` - or `docker exec -t kafka1 /usr/bin/kafka-console-consumer --bootstrap-server localhost:9092 --topic aa --from-beginning` From 0dfd381246a7acd35e64e0a88f1a01b3c93be1f2 Mon Sep 17 00:00:00 2001 From: jist <95856749+george0st@users.noreply.github.com> Date: Sat, 30 Mar 2024 13:02:58 +0100 Subject: [PATCH 4/6] Update kafka.md --- docs/kafka.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/kafka.md b/docs/kafka.md index a11777e9..8669e8ed 100644 --- a/docs/kafka.md +++ b/docs/kafka.md @@ -37,6 +37,13 @@ - format `QGATE_KAFKA = , ` - see `QGATE_KAFKA = localhost:9092, testtopic` +## 4. Install these python packages + + - SQLAlchemy based on MLRun extras see `pip install mlrun[kafka]` or [dependencies.py in MLRun](https://github.com/mlrun/mlrun/blob/development/dependencies.py) + - it required `pip install kafka-python~=2.0` + - it required `pip install avro~=1.11` + - NOTE + - these are versions valid for MLRun 1.6.1 ## Useful sources for kafka From 316bb0ce67cd8baf8d2a5fcfe03281844fd649ec Mon Sep 17 00:00:00 2001 From: jist <95856749+george0st@users.noreply.github.com> Date: Sat, 30 Mar 2024 13:03:18 +0100 Subject: [PATCH 5/6] Update kafka.md --- docs/kafka.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/kafka.md b/docs/kafka.md index 8669e8ed..da99637c 100644 --- a/docs/kafka.md +++ b/docs/kafka.md @@ -43,7 +43,7 @@ - it required `pip install kafka-python~=2.0` - it required `pip install avro~=1.11` - NOTE - - these are versions valid for MLRun 1.6.1 + - these are versions valid for MLRun 1.6.2 ## Useful sources for kafka From 27b9fb89ae96cc7184a719368fdee32a77210c57 Mon Sep 17 00:00:00 2001 From: jist <95856749+george0st@users.noreply.github.com> Date: Sat, 30 Mar 2024 13:03:55 +0100 Subject: [PATCH 6/6] ... --- docs/mysql.md | 2 +- docs/postgres.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/mysql.md b/docs/mysql.md index 382707dc..eb48b530 100644 --- a/docs/mysql.md +++ b/docs/mysql.md @@ -51,5 +51,5 @@ - Cryptography support - it required `pip install cryptography~=42.0` - NOTE - - these are versions valid for MLRun 1.6.1 + - these are versions valid for MLRun 1.6.2 diff --git a/docs/postgres.md b/docs/postgres.md index 8afcb775..498294c0 100644 --- a/docs/postgres.md +++ b/docs/postgres.md @@ -53,4 +53,4 @@ - Cryptography support - it required `pip install cryptography~=42.0` - NOTE - - these are versions valid for MLRun 1.6.1 + - these are versions valid for MLRun 1.6.2