These are the initial steps used in installing Hashicorp Vault as a prerequisite for installing TM Vault. This is a document for COLLABORATION. DevOps/SRE team members are allowed to edit and improve this.
Connect via ssh console to TM bastion host, install helm and install hashicorp vault.
Reference: https://learn.hashicorp.com/tutorials/vault/kubernetes-google-cloud-gke
# Install helm-cli
veneraldo@bastion-safi-sandbox-tm1-2md0:~$ curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
veneraldo@bastion-safi-sandbox-tm1-2md0:~$ chmod 700 get_helm.sh
veneraldo@bastion-safi-sandbox-tm1-2md0:~$ ./get_helm.sh
Downloading https://get.helm.sh/helm-v3.9.0-linux-amd64.tar.gz
Verifying checksum... Done.
Preparing to install helm into /usr/local/bin
helm installed into /usr/local/bin/helm
veneraldo@bastion-safi-sandbox-tm1-2md0:~$ helm version
version.BuildInfo{Version:"v3.9.0", GitCommit:"7ceeda6c585217a19a1131663d8cd1f7d641b2a7", GitTreeState:"clean", GoVersion:"go1.17.5"}
veneraldo@bastion-safi-sandbox-tm1-2md0:~$ helm repo add hashicorp https://helm.releases.hashicorp.com
"hashicorp" has been added to your repositories
veneraldo@bastion-safi-sandbox-tm1-2md0:~$ helm repo update
Hang tight while we grab the latest from your chart repositories...
...Successfully got an update from the "hashicorp" chart repository
Update Complete. ⎈Happy Helming!⎈
veneraldo@bastion-safi-sandbox-tm1-2md0:~$ helm search repo vault --versions
NAME CHART VERSION APP VERSION DESCRIPTION
hashicorp/vault 0.20.0 1.10.3 Official HashiCorp Vault Chart
hashicorp/vault 0.19.0 1.9.2 Official HashiCorp Vault Chart
hashicorp/vault 0.18.0 1.9.0 Official HashiCorp Vault Chart
hashicorp/vault 0.17.1 1.8.4 Official HashiCorp Vault Chart
hashicorp/vault 0.17.0 1.8.4 Official HashiCorp Vault Chart
hashicorp/vault 0.16.1 1.8.3 Official HashiCorp Vault Chart
hashicorp/vault 0.16.0 1.8.2 Official HashiCorp Vault Chart
hashicorp/vault 0.15.0 1.8.1 Official HashiCorp Vault Chart
hashicorp/vault 0.14.0 1.8.0 Official HashiCorp Vault Chart
hashicorp/vault 0.13.0 1.7.3 Official HashiCorp Vault Chart
hashicorp/vault 0.12.0 1.7.2 Official HashiCorp Vault Chart
hashicorp/vault 0.11.0 1.7.0 Official HashiCorp Vault Chart
hashicorp/vault 0.10.0 1.7.0 Official HashiCorp Vault Chart
hashicorp/vault 0.9.1 1.6.2 Official HashiCorp Vault Chart
hashicorp/vault 0.9.0 1.6.1 Official HashiCorp Vault Chart
hashicorp/vault 0.8.0 1.5.4 Official HashiCorp Vault Chart
hashicorp/vault 0.7.0 1.5.2 Official HashiCorp Vault Chart
hashicorp/vault 0.6.0 1.4.2 Official HashiCorp Vault Chart
hashicorp/vault 0.5.0 Install and configure Vault on Kubernetes.
hashicorp/vault 0.4.0 Install and configure Vault on Kubernetes.
# Create a namespace for hashicorp vault
veneraldo@bastion-safi-sandbox-tm1-2md0:~$ kubectl create namespace hashicorp-vault
namespace/hashicorp-vault created
# Install the latest version of the vault helm chart in HA mode with integrated storage
veneraldo@bastion-safi-sandbox-tm1-2md0:~$ helm install vault hashicorp/vault -n hashicorp-vault \
> --set='server.ha.enabled=true' \
> --set='server.ha.raft.enabled=true'
# Show the pods in the namespace
veneraldo@bastion-safi-sandbox-tm1-2md0:~$ kubectl -n hashicorp-vault get pods
NAME READY STATUS RESTARTS AGE
vault1-0 0/1 Running 0 36m
vault1-1 0/1 Running 0 41s
vault1-2 0/1 ContainerCreating 0 11s
vault1-agent-injector-8596548968-ch8dq 1/1 Running 0 36m
# The vault1-0 up to vault1-2 are running but all not yet ready i.e. 0/1 because we still need another steps to follow
# Check one vault pod status
veneraldo@bastion-safi-sandbox-tm1-2md0:~$ kubectl -n hashicorp-vault exec vault1-0 -- vault status
Key Value
--- -----
Seal Type shamir
Initialized false
Sealed true
Total Shares 0
Threshold 0
Unseal Progress 0/0
Unseal Nonce n/a
Version 1.10.3
Storage Type file
HA Enabled false
command terminated with exit code 2
# Initialize vault with one key share and one key threshold
veneraldo@bastion-safi-sandbox-tm1-2md0:~$ kubectl -n hashicorp-vault exec vault1-0 -- vault operator init -key-shares=1 -key-threshold=1 -format=json > cluster-keys.json
# Check the pod status again
veneraldo@bastion-safi-sandbox-tm1-2md0:~$ kubectl -n hashicorp-vault exec vault1-0 -- vault status
Key Value
--- -----
Seal Type shamir
Initialized true
Sealed true
Total Shares 1
Threshold 1
Unseal Progress 0/1
Unseal Nonce n/a
Version 1.10.3
Storage Type file
HA Enabled false
command terminated with exit code 2
# Display the unseal key found in cluster-keys.json.
veneraldo@bastion-safi-sandbox-tm1-2md0:~$ cat cluster-keys.json | jq -r ".unseal_keys_b64[]"
wxf3XbjhWvUV/1If7D7hUEnIVwuHTkxcyknrrMIPe84=
# Create a variable named VAULT_UNSEAL_KEY to capture the Vault unseal key
veneraldo@bastion-safi-sandbox-tm1-2md0:~$ VAULT_UNSEAL_KEY=$(cat cluster-keys.json | jq -r ".unseal_keys_b64[]")
# Unseal Vault running on the vault1-0 pod
veneraldo@bastion-safi-sandbox-tm1-2md0:~$ kubectl -n hashicorp-vault exec vault1-0 -- vault operator unseal $VAULT_UNSEAL_KEY
Key Value
--- -----
Seal Type shamir
Initialized true
Sealed false
Total Shares 1
Threshold 1
Version 1.10.3
Storage Type file
Cluster Name vault-cluster-b3b2bef6
Cluster ID 96fce27e-d031-4ccd-32ab-bc7a87be937f
HA Enabled false
# Verify if the pod vault1-0 is in READY state
veneraldo@bastion-safi-sandbox-tm1-2md0:~$ kubectl -n hashicorp-vault get pods
NAME READY STATUS RESTARTS AGE
vault1-0 1/1 Running 0 43m
vault1-1 0/1 Running 0 7m40s
vault1-2 0/1 Running 0 7m10s
vault1-agent-injector-8596548968-ch8dq 1/1 Running 0 43m
# Check if if the Vault is working
veneraldo@bastion-safi-sandbox-tm1-2md0:~$ kubectl -n hashicorp-vault exec --stdin=true --tty=true vault1-0 -- /bin/sh
/ $ vault auth enable kubernetes
Success! Enabled kubernetes auth method at: kubernetes/
/ $ vault secrets enable -path=secret kv-v2
Success! Enabled the kv-v2 secrets engine at: secret/
/ $ vault kv put secret/devwebapp/config username='darth' password='vader'
======== Secret Path ========
secret/data/devwebapp/config
======= Metadata =======
Key Value
--- -----
created_time 2022-05-20T07:29:29.071817312Z
custom_metadata <nil>
deletion_time n/a
destroyed false
version 1
/ $ vault kv get secret/devwebapp/config
======== Secret Path ========
secret/data/devwebapp/config
======= Metadata =======
Key Value
--- -----
created_time 2022-05-20T07:29:29.071817312Z
custom_metadata <nil>
deletion_time n/a
destroyed false
version 1
====== Data ======
Key Value
--- -----
password vader
username darth
/ $ exit
veneraldo@bastion-safi-sandbox-tm1-2md0:~$ kubectl -n hashicorp-vault get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
vault1-0 1/1 Running 0 22h 172.20.2.35 gke-safi-sandbox-tm1-default-safi-san-2651d92e-04m2 <none> <none>
vault1-1 0/1 Running 0 21h 172.20.0.14 gke-safi-sandbox-tm1-default-safi-san-54bcd53b-l3qg <none> <none>
vault1-2 0/1 Running 0 21h 172.20.1.10 gke-safi-sandbox-tm1-default-safi-san-e4c237c1-8jsx <none> <none>
vault1-agent-injector-8596548968-ch8dq 1/1 Running 0 22h 172.20.2.34 gke-safi-sandbox-tm1-default-safi-san-2651d92e-04m2 <none> <none>
veneraldo@bastion-safi-sandbox-tm1-2md0:~$ kubectl -n hashicorp-vault exec vault1-0 -- printenv | grep SERVICE
KUBERNETES_SERVICE_PORT_HTTPS=443
VAULT1_SERVICE_PORT_HTTPS_INTERNAL=8201
VAULT1_AGENT_INJECTOR_SVC_SERVICE_PORT=443
VAULT1_AGENT_INJECTOR_SVC_SERVICE_PORT_HTTPS=443
KUBERNETES_SERVICE_HOST=172.18.0.1
VAULT1_SERVICE_PORT=8200
VAULT1_SERVICE_PORT_HTTP=8200
KUBERNETES_SERVICE_PORT=443
VAULT1_AGENT_INJECTOR_SVC_SERVICE_HOST=172.19.193.84
VAULT1_SERVICE_HOST=172.19.10.25
veneraldo@bastion-safi-sandbox-tm1-2md0:~/installation$ kubectl -n hashicorp-vault get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
vault1 ClusterIP 172.19.10.25 <none> 8200/TCP,8201/TCP 23h
vault1-active ClusterIP 172.18.238.15 <none> 8200/TCP,8201/TCP 23h
vault1-agent-injector-svc ClusterIP 172.19.193.84 <none> 443/TCP 23h
vault1-internal ClusterIP None <none> 8200/TCP,8201/TCP 23h
vault1-standby ClusterIP 172.19.14.196 <none> 8200/TCP,8201/TCP 23h
# Use the IP 172.19.193.84 as value in TM values.yml i.e. replace vault.dev.your.bank.com
veneraldo@bastion-safi-sandbox-tm1-2md0:~/installation$ cat values.yaml | grep address
# > address - The Hashicorp Vault endpoint
address: https://vault.dev.your.bank.com
# Resume the installation of TM the kafka part
# Copy values.yaml into /release-artifacts/config-templates/ within the pod:
veneraldo@bastion-safi-sandbox-tm1-2md0:~/installation$ kubectl -n tm-system cp values.yaml \
vault-installer-tm-system:/release-artifacts/config-templates/
# Run the TM Vault Installer which is totally different from HashiCorp Vault ... Crikey me...what the..
veneraldo@bastion-safi-sandbox-tm1-2md0:~/installation$ kubectl exec -it -n tm-system vault-installer-tm-system -- /deployment-tools/install-vault --component kafka --dry_run
ERROR RESULT:
Traceback (most recent call last):
File "/deployment-tools/install-vault/third_party/python3/urllib3/util/ssl_.py", line 402, in ssl_wrap_socket
ssl.SSLError: [X509] PEM lib (_ssl.c:4293)
requests.exceptions.SSLError: HTTPSConnectionPool(host='172.19.193.84', port=443): Max retries exceeded with url: /v1/auth/token/lookup-self (Caused by SSLError(SSLError(9, '[X509] PEM lib (_ssl.c:4293)')))
command terminated with exit code 1
# We need to paste the self sign certificate :-(