diff --git a/docker/ELK.sh b/docker/ELK.sh new file mode 100644 index 0000000..f1d8b17 --- /dev/null +++ b/docker/ELK.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +docker run -d -p 9200:9200 -p 5044:5044 -p 5000:5000 -it -h elasticsearch --name elasticsearch elasticsearch + +docker run -d -p 5601:5601 -h kibana --name kibana --link elasticsearch:elasticsearch kibana + +docker run -p 9500:9500 -h logstash --name logstash --link elasticsearch:elasticsearch -it --rm -v "$PWD":/config-dir logstash -f /config-dir/logstash.conf diff --git a/docker/jupyter/Dockerfile b/docker/jupyter/Dockerfile index c8848d2..52c0373 100644 --- a/docker/jupyter/Dockerfile +++ b/docker/jupyter/Dockerfile @@ -3,8 +3,8 @@ #Tested on tensorflow/tensorflow and jupyter/datascience-notebook FROM tensorflow/tensorflow:latest -# Install required libraries for working with deep learning -RUN pip install keras seaborn h5py +# Install required libraries for working with machine learing, deep learning and visualisation +RUN pip install keras seaborn h5py haversine plotly bokeh # Install spacy (NLP) and download for english language RUN pip install -U spacy diff --git a/others/Kafka-MirrorMaker/docker-setup-kafka-mirrormaker.sh b/others/Kafka-MirrorMaker/docker-setup-kafka-mirrormaker.sh new file mode 100755 index 0000000..a6d33e3 --- /dev/null +++ b/others/Kafka-MirrorMaker/docker-setup-kafka-mirrormaker.sh @@ -0,0 +1,27 @@ +#!/bin/bash +docker run -d -it --name zookeeper-source \ + -h zookeeper-source \ + -p 2181:2181 \ + confluent/zookeeper + +docker run -d -it --name zookeeper-dest \ + -h zookeeper-dest \ + -p 2182:2181 \ + confluent/zookeeper + +docker run -d -it --name kafka-source \ + -h kafka-source \ + --env KAFKA_ADVERTISED_HOST_NAME=kafka-source \ + -p 9092:9092 -p 9093:9033 \ + --link zookeeper-source:zookeeper \ + confluent/kafka:0.10.0.0-cp1 + +docker run -d -it --name kafka-dest \ + -h kafka-dest \ + --env KAFKA_ADVERTISED_HOST_NAME=kafka-dest \ + -p 9094:9092 -p 9095:9033 \ + --link zookeeper-dest:zookeeper \ + --link zookeeper-source:zookeeper-source \ + --link kafka-source:kafka-source \ + -v $PWD:/opt/software \ + confluent/kafka:0.10.0.0-cp1 \ No newline at end of file diff --git a/others/Kafka-MirrorMaker/sourceClusterConsumer.properties b/others/Kafka-MirrorMaker/sourceClusterConsumer.properties new file mode 100644 index 0000000..835c5f6 --- /dev/null +++ b/others/Kafka-MirrorMaker/sourceClusterConsumer.properties @@ -0,0 +1,5 @@ +zookeeper.connect=zookeeper-source:2181 +zookeeper.connectiontimeout.ms=1000000 +consumer.timeout.ms=-1 +group.id=dp-MirrorMaker-test-datap1 +shallow.iterator.enable=true \ No newline at end of file diff --git a/others/Kafka-MirrorMaker/targetClusterProducer.properties b/others/Kafka-MirrorMaker/targetClusterProducer.properties new file mode 100644 index 0000000..a243e11 --- /dev/null +++ b/others/Kafka-MirrorMaker/targetClusterProducer.properties @@ -0,0 +1,8 @@ +zk.connect=zookeeper-dest:2181 +producer.type=async +compression.codec=0 +serializer.class=kafka.serializer.DefaultEncoder +max.message.size=10000000 +queue.time=1000 +queue.enqueueTimeout.ms=-1 +bootstrap.servers=kafka-dest:9092 \ No newline at end of file diff --git a/others/Kafka-MirrorMaker/test-mirrormaker.sh b/others/Kafka-MirrorMaker/test-mirrormaker.sh new file mode 100755 index 0000000..b1732f0 --- /dev/null +++ b/others/Kafka-MirrorMaker/test-mirrormaker.sh @@ -0,0 +1,11 @@ +#!/bin/bash +kafka-topics --list --zookeeper zookeeper-source:2181 +kafka-topics --list --zookeeper zookeeper-dest:2181 +kafka-topics --list --zookeeper zookeeper-source:2181 + +kafka-topics --create --zookeeper zookeeper-source:2181 --replication-factor 1 --partitions 5 --topic Test1 +kafka-console-producer --broker-list localhost:9092 --topic Test1 + +kafka-mirror-maker --consumer.config sourceClusterConsumer.properties --producer.config targetClusterProducer.properties --whitelist=".*" + +kafka-console-consumer --zookeeper zookeeper-dest:2181 --topic Test1 --from-beginning \ No newline at end of file diff --git a/others/common-ssh-via-bastion.cfg b/others/common-ssh-via-bastion.cfg new file mode 100644 index 0000000..7ddcb1d --- /dev/null +++ b/others/common-ssh-via-bastion.cfg @@ -0,0 +1,23 @@ +#Connect using: ssh -F ~/.ssh/common-ssh-via-bastion.cfg + +ServerAliveInterval 120 +StrictHostKeyChecking no +UserKnownHostsFile /dev/null + +User +Host bastion1 + Hostname + ProxyCommand none + +Host + Hostname + ProxyCommand ssh -F ~/.ssh/common-ssh-via-bastion.cfg -o ControlMaster=auto -o ControlPersist=60s bastion1 nc -q0 %h %p + LocalForward 7183 :7183 + LocalForward 8000 :80 + LocalForward 8161 :8161 + LocalForward 5601 :5601 + LocalForward 8787 :8787 + LocalForward 8888 :8888 + LocalForward 8088 :8088 + LocalForward 8090 :8090 + diff --git a/others/working with kerberos.md b/others/working with kerberos.md new file mode 100644 index 0000000..7f72a6b --- /dev/null +++ b/others/working with kerberos.md @@ -0,0 +1,47 @@ +# Useful commands when working with kerberos + + +## Create a key tab file +On kdc run + +``` +ktutil +ktutil: addent -password -p @ -k 1 -e aes256-cts +(OUTPUT) Password for @: +ktutil: wkt .keytab +ktutil: quit +``` + + +# kinit + +Dont ever do this on prod :) +``` +echo | kinit +``` + + +## List Principals in Keytab + +Command should provide output like +Keytab name: FILE:.keytab +KVNO Principal +---- -------------------------------------------------------------------------- + 1 @ (aes256-cts-hmac-sha1-96) + +``` +klist -e -k .keytab +``` + +## kinit via keytab file + +``` +kinit -k -t .keytab +``` + +kinit and run some command + +``` +kinit -k -t /opt/keytab//.keytab; /usr/bin/curl -k --negotiate -u: https://<>/ +``` +