General

docker/ELK.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+docker run -d -p 9200:9200 -p 5044:5044 -p 5000:5000 -it -h elasticsearch --name elasticsearch elasticsearch
+
+docker run -d  -p 5601:5601 -h kibana --name kibana --link elasticsearch:elasticsearch kibana
+
+docker run -p 9500:9500  -h logstash --name logstash --link elasticsearch:elasticsearch -it --rm -v "$PWD":/config-dir logstash -f /config-dir/logstash.conf

docker/jupyter/Dockerfile

@@ -3,8 +3,8 @@
 #Tested on tensorflow/tensorflow and jupyter/datascience-notebook
 FROM tensorflow/tensorflow:latest
 
-# Install required libraries for working with deep learning
-RUN pip install keras seaborn h5py
+# Install required libraries for working with machine learing, deep learning and visualisation
+RUN pip install keras seaborn h5py haversine plotly bokeh
 
 # Install spacy (NLP) and download for english language
 RUN pip install -U spacy

others/Kafka-MirrorMaker/docker-setup-kafka-mirrormaker.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+docker run -d -it --name zookeeper-source \
+        -h zookeeper-source \
+        -p 2181:2181 \
+        confluent/zookeeper
+
+docker run -d -it --name zookeeper-dest \
+        -h zookeeper-dest \
+        -p 2182:2181 \
+        confluent/zookeeper
+
+docker run -d -it --name kafka-source \
+        -h kafka-source \
+        --env KAFKA_ADVERTISED_HOST_NAME=kafka-source \
+        -p 9092:9092  -p 9093:9033 \
+        --link zookeeper-source:zookeeper \
+        confluent/kafka:0.10.0.0-cp1
+
+docker run -d -it --name kafka-dest \
+        -h kafka-dest \
+        --env KAFKA_ADVERTISED_HOST_NAME=kafka-dest \
+        -p 9094:9092  -p 9095:9033 \
+        --link zookeeper-dest:zookeeper \
+        --link zookeeper-source:zookeeper-source \
+        --link kafka-source:kafka-source \
+        -v $PWD:/opt/software \
+        confluent/kafka:0.10.0.0-cp1

others/Kafka-MirrorMaker/sourceClusterConsumer.properties

@@ -0,0 +1,5 @@
+zookeeper.connect=zookeeper-source:2181
+zookeeper.connectiontimeout.ms=1000000
+consumer.timeout.ms=-1
+group.id=dp-MirrorMaker-test-datap1
+shallow.iterator.enable=true

others/Kafka-MirrorMaker/targetClusterProducer.properties

@@ -0,0 +1,8 @@
+zk.connect=zookeeper-dest:2181
+producer.type=async
+compression.codec=0
+serializer.class=kafka.serializer.DefaultEncoder
+max.message.size=10000000
+queue.time=1000
+queue.enqueueTimeout.ms=-1
+bootstrap.servers=kafka-dest:9092

others/Kafka-MirrorMaker/test-mirrormaker.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+kafka-topics --list --zookeeper zookeeper-source:2181
+kafka-topics --list --zookeeper zookeeper-dest:2181
+kafka-topics --list --zookeeper zookeeper-source:2181
+
+kafka-topics --create --zookeeper zookeeper-source:2181  --replication-factor 1 --partitions 5 --topic Test1
+kafka-console-producer --broker-list localhost:9092 --topic Test1
+
+kafka-mirror-maker --consumer.config sourceClusterConsumer.properties  --producer.config targetClusterProducer.properties --whitelist=".*"
+
+kafka-console-consumer --zookeeper zookeeper-dest:2181 --topic Test1 --from-beginning

others/common-ssh-via-bastion.cfg

@@ -0,0 +1,23 @@
+#Connect using: ssh -F ~/.ssh/common-ssh-via-bastion.cfg <host name>
+
+ServerAliveInterval 120
+StrictHostKeyChecking no
+UserKnownHostsFile /dev/null
+
+User <melvin>
+Host bastion1
+  Hostname <bastion url>
+  ProxyCommand none
+
+Host <edgenode/other>
+  Hostname <ip>
+  ProxyCommand ssh -F ~/.ssh/common-ssh-via-bastion.cfg -o ControlMaster=auto -o ControlPersist=60s bastion1 nc -q0 %h %p
+  LocalForward 7183 <cm-manager>:7183
+  LocalForward 8000 <jupyterhub>:80
+  LocalForward 8161 <activemq>:8161
+  LocalForward 5601 <kibana>:5601
+  LocalForward 8787 <rstudio>:8787
+  LocalForward 8888 <hue>:8888
+  LocalForward 8088 <jupyterhub>:8088
+  LocalForward 8090 <yarn-rm>:8090
+

others/working with kerberos.md

@@ -0,0 +1,47 @@
+# Useful commands when working with kerberos
+
+
+## Create a key tab file 
+On kdc run 
+
+```
+ktutil
+ktutil:  addent -password -p <user>@<full doman name> -k 1 -e aes256-cts
+(OUTPUT) Password for  <user>@<full doman name>:
+ktutil:  wkt <user name>.keytab
+ktutil:  quit
+```
+
+
+# kinit 
+
+Dont ever do this on prod :)  
+``` 
+echo <password> | kinit <username>
+```
+
+
+## List Principals in Keytab
+
+Command should provide output like 
+Keytab name: FILE:<user name>.keytab
+KVNO Principal
+---- --------------------------------------------------------------------------
+   1 <user>@<full doman name> (aes256-cts-hmac-sha1-96)
+
+```
+klist -e -k <user name>.keytab
+```
+
+## kinit via keytab file 
+
+```
+kinit <user name> -k -t <user name>.keytab
+```
+
+kinit and run some command 
+
+```
+kinit <user> -k -t /opt/keytab/<user>/<user>.keytab; /usr/bin/curl -k --negotiate  -u: https://<>/
+```
+