giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial stage

This commit is contained in:
Giambattista Bloisi 2024-03-21 12:23:04 +01:00
parent 390d07747d
commit f5fa8d7a13
35 changed files with 2895 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
PREREQUISITES.txt Normal file
View File

@ -0,0 +1,75 @@
https://kluctl.io/
kubectl
AIRFLOW
AIRBYTE
https://medium.com/apache-airflow/what-we-learned-after-running-airflow-on-kubernetes-for-2-years-0537b157acfd
https://github.com/opensearch-project/helm-charts/blob/main/README.md#installing
SETUP
kind create cluster --config clusters/local/kind-cluster-config.yaml
./clusters/local/kind-with-registry.sh kind-openaire-data-platform
kubectl apply --context kind-openaire-data-platform -f ./clusters/local/nginx-kind-deploy.yaml
Using The Registry
The registry can be used like this.
First we’ll pull an image docker pull gcr.io/google-samples/hello-app:1.0
Then we’ll tag the image to use the local registry docker tag gcr.io/google-samples/hello-app:1.0 localhost:5001/hello-app:1.0
Then we’ll push it to the registry docker push localhost:5001/hello-app:1.0
And now we can use the image kubectl create deployment hello-server --image=localhost:5001/hello-app:1.0
If you build your own image and tag it like localhost:5001/image:foo and then use it in kubernetes as localhost:5001/image:foo. And use it from inside of your cluster application as kind-registry:5000.
CLEANUP
kind delete cluster --name openaire-data-platform
OPENASEARCH OPERATOR
HELM
kubectl use-context openaire-data-platform
helm repo add opensearch https://opensearch-project.github.io/helm-charts/
helm repo update
helm upgrade --install oa-opensearch opensearch/opensearch --version 2.17.2 -f envs/local/common.yaml -f envs/local/opensearch.yaml
opensearch/opensearch 2.17.2 2.11.1 A Helm chart for OpenSearch
opensearch/opensearch-dashboards 2.15.1 2.11.1 A Helm chart for OpenSearch Dashboards
helm repo add jetstack https://charts.jetstack.io
helm repo update
helm install \
cert-manager jetstack/cert-manager \
--namespace cert-manager \
--create-namespace \
--version v1.14.4 \
--set installCRDs=true \
--set global.leaderElection.namespace=cert-manager
➜ dataplatform kubectl apply -f envs/gcp/letsencrypt-prod.yaml
clusterissuer.cert-manager.io/letsencrypt-prod created
➜ dataplatform kubectl apply -f envs/gcp/duckdnsdomain.yaml
certificate.cert-manager.io/openaire-duckdns created
kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.4/cert-manager.yaml
kubectl apply -f envs/gcp/local-path-storage.yaml
helm repo add opensearch-operator https://opensearch-project.github.io/opensearch-k8s-operator/
helm install opensearch-operator opensearch-operator/opensearch-operator --version 2.5.1 -f envs/gcp/opensearch-operator.yaml
kubectl port-forward svc/opensearch-cluster-dashboards 5601 -n oa-opensearch
helm repo add airbyte https://airbytehq.github.io/helm-charts/airbyte
helm upgrade --install oa-airbyte airbyte/airbyte --namespace oa-airbyte --create-namespace --version 0.53.196 -f envs/local/common.yaml -f envs/local/airbyte.yaml

27
clusters/local/kind-cluster-config.yaml Normal file
View File

@ -0,0 +1,27 @@
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
name: openaire-data-platform
nodes:
- role: control-plane
image: kindest/node:v1.28.6@sha256:e9e59d321795595d0eed0de48ef9fbda50388dc8bd4a9b23fb9bd869f370ec7e
kubeadmConfigPatches:
- |
kind: InitConfiguration
nodeRegistration:
kubeletExtraArgs:
node-labels: "ingress-ready=true"
authorization-mode: "AlwaysAllow"
extraPortMappings:
- containerPort: 80
hostPort: 80
protocol: TCP
- containerPort: 443
hostPort: 443
protocol: TCP
containerdConfigPatches:
- |-
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d"

32
clusters/local/kind-with-registry.sh Executable file
View File

@ -0,0 +1,32 @@
#!/bin/sh
set -o errexit
# Script Origin: https://kind.sigs.k8s.io/docs/user/local-registry/
# create registry container unless it already exists
reg_name='kind-registry'
reg_port='5001'
if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
docker run \
-d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \
registry:2
fi
# connect the registry to the cluster network if not already connected
if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
docker network connect "kind" "${reg_name}"
fi
# Document the local registry
# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
cat <<EOF | kubectl apply --context $1 -f -
apiVersion: v1
kind: ConfigMap
metadata:
name: local-registry-hosting
namespace: kube-public
data:
localRegistryHosting.v1: |
host: "localhost:${reg_port}"
help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
EOF

671
clusters/local/nginx-kind-deploy.yaml Normal file
View File

@ -0,0 +1,671 @@
apiVersion: v1
kind: Namespace
metadata:
labels:
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx
---
apiVersion: v1
automountServiceAccountToken: true
kind: ServiceAccount
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.9.6
name: ingress-nginx
namespace: ingress-nginx
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.9.6
name: ingress-nginx-admission
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.9.6
name: ingress-nginx
namespace: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- namespaces
verbs:
- get
- apiGroups:
- ""
resources:
- configmaps
- pods
- secrets
- endpoints
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- networking.k8s.io
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- networking.k8s.io
resources:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io
resources:
- ingressclasses
verbs:
- get
- list
- watch
- apiGroups:
- coordination.k8s.io
resourceNames:
- ingress-nginx-leader
resources:
- leases
verbs:
- get
- update
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs:
- create
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- apiGroups:
- discovery.k8s.io
resources:
- endpointslices
verbs:
- list
- watch
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.9.6
name: ingress-nginx-admission
namespace: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.9.6
name: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- configmaps
- endpoints
- nodes
- pods
- secrets
- namespaces
verbs:
- list
- watch
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- networking.k8s.io
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- apiGroups:
- networking.k8s.io
resources:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io
resources:
- ingressclasses
verbs:
- get
- list
- watch
- apiGroups:
- discovery.k8s.io
resources:
- endpointslices
verbs:
- list
- watch
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.9.6
name: ingress-nginx-admission
rules:
- apiGroups:
- admissionregistration.k8s.io
resources:
- validatingwebhookconfigurations
verbs:
- get
- update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.9.6
name: ingress-nginx
namespace: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: ingress-nginx
subjects:
- kind: ServiceAccount
name: ingress-nginx
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.9.6
name: ingress-nginx-admission
namespace: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: ingress-nginx-admission
subjects:
- kind: ServiceAccount
name: ingress-nginx-admission
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.9.6
name: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: ingress-nginx
subjects:
- kind: ServiceAccount
name: ingress-nginx
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.9.6
name: ingress-nginx-admission
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: ingress-nginx-admission
subjects:
- kind: ServiceAccount
name: ingress-nginx-admission
namespace: ingress-nginx
---
apiVersion: v1
data:
allow-snippet-annotations: "false"
kind: ConfigMap
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.9.6
name: ingress-nginx-controller
namespace: ingress-nginx
---
apiVersion: v1
kind: Service
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.9.6
name: ingress-nginx-controller
namespace: ingress-nginx
spec:
ipFamilies:
- IPv4
ipFamilyPolicy: SingleStack
ports:
- appProtocol: http
name: http
port: 80
protocol: TCP
targetPort: http
- appProtocol: https
name: https
port: 443
protocol: TCP
targetPort: https
selector:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
type: NodePort
---
apiVersion: v1
kind: Service
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.9.6
name: ingress-nginx-controller-admission
namespace: ingress-nginx
spec:
ports:
- appProtocol: https
name: https-webhook
port: 443
targetPort: webhook
selector:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
type: ClusterIP
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.9.6
name: ingress-nginx-controller
namespace: ingress-nginx
spec:
minReadySeconds: 0
revisionHistoryLimit: 10
selector:
matchLabels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
strategy:
rollingUpdate:
maxUnavailable: 1
type: RollingUpdate
template:
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.9.6
spec:
containers:
- args:
- /nginx-ingress-controller
- --election-id=ingress-nginx-leader
- --controller-class=k8s.io/ingress-nginx
- --ingress-class=nginx
- --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
- --validating-webhook=:8443
- --validating-webhook-certificate=/usr/local/certificates/cert
- --validating-webhook-key=/usr/local/certificates/key
- --watch-ingress-without-class=true
- --publish-status-address=localhost
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: LD_PRELOAD
value: /usr/local/lib/libmimalloc.so
image: registry.k8s.io/ingress-nginx/controller:v1.9.6@sha256:1405cc613bd95b2c6edd8b2a152510ae91c7e62aea4698500d23b2145960ab9c
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- /wait-shutdown
livenessProbe:
failureThreshold: 5
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
name: controller
ports:
- containerPort: 80
hostPort: 80
name: http
protocol: TCP
- containerPort: 443
hostPort: 443
name: https
protocol: TCP
- containerPort: 8443
name: webhook
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
resources:
requests:
cpu: 100m
memory: 90Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- NET_BIND_SERVICE
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: true
runAsUser: 101
seccompProfile:
type: RuntimeDefault
volumeMounts:
- mountPath: /usr/local/certificates/
name: webhook-cert
readOnly: true
dnsPolicy: ClusterFirst
nodeSelector:
ingress-ready: "true"
kubernetes.io/os: linux
serviceAccountName: ingress-nginx
terminationGracePeriodSeconds: 0
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
operator: Equal
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
operator: Equal
volumes:
- name: webhook-cert
secret:
secretName: ingress-nginx-admission
---
apiVersion: batch/v1
kind: Job
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.9.6
name: ingress-nginx-admission-create
namespace: ingress-nginx
spec:
template:
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.9.6
name: ingress-nginx-admission-create
spec:
containers:
- args:
- create
- --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc
- --namespace=$(POD_NAMESPACE)
- --secret-name=ingress-nginx-admission
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20231226-1a7112e06@sha256:25d6a5f11211cc5c3f9f2bf552b585374af287b4debf693cacbe2da47daa5084
imagePullPolicy: IfNotPresent
name: create
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 65532
seccompProfile:
type: RuntimeDefault
nodeSelector:
kubernetes.io/os: linux
restartPolicy: OnFailure
serviceAccountName: ingress-nginx-admission
---
apiVersion: batch/v1
kind: Job
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.9.6
name: ingress-nginx-admission-patch
namespace: ingress-nginx
spec:
template:
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.9.6
name: ingress-nginx-admission-patch
spec:
containers:
- args:
- patch
- --webhook-name=ingress-nginx-admission
- --namespace=$(POD_NAMESPACE)
- --patch-mutating=false
- --secret-name=ingress-nginx-admission
- --patch-failure-policy=Fail
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20231226-1a7112e06@sha256:25d6a5f11211cc5c3f9f2bf552b585374af287b4debf693cacbe2da47daa5084
imagePullPolicy: IfNotPresent
name: patch
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 65532
seccompProfile:
type: RuntimeDefault
nodeSelector:
kubernetes.io/os: linux
restartPolicy: OnFailure
serviceAccountName: ingress-nginx-admission
---
apiVersion: networking.k8s.io/v1
kind: IngressClass
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.9.6
name: nginx
spec:
controller: k8s.io/ingress-nginx
---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.9.6
name: ingress-nginx-admission
webhooks:
- admissionReviewVersions:
- v1
clientConfig:
service:
name: ingress-nginx-controller-admission
namespace: ingress-nginx
path: /networking/v1/ingresses
failurePolicy: Fail
matchPolicy: Equivalent
name: validate.nginx.ingress.kubernetes.io
rules:
- apiGroups:
- networking.k8s.io
apiVersions:
- v1
operations:
- CREATE
- UPDATE
resources:
- ingresses
sideEffects: None

7
docker-images/airflow/Dockerfile Executable file
View File

@ -0,0 +1,7 @@
#!/usr/bin/env -S docker build . --tag=gbloisi/airflow:2.8.3rc1-python3.11 --platform linux/arm64/v8,linux/amd64 --push --network=host --file
FROM apache/airflow:2.8.3rc1-python3.11
COPY requirements.txt /
RUN pip install --no-cache-dir "apache-airflow==${AIRFLOW_VERSION}" -r /requirements.txt

14
docker-images/airflow/requirements.txt Normal file
View File

@ -0,0 +1,14 @@
apache-airflow-providers-amazon
apache-airflow-providers-apache-spark
apache-airflow-providers-cncf-kubernetes
apache-airflow-providers-opensearch
apache-airflow-providers-postgres
apache-airflow-providers-sftp
apache-airflow[google]
msgspec
opensearch-py
opensearch-py-ml
smart_open[all]

2
docker-images/airflow/upload.sh Normal file
View File

@ -0,0 +1,2 @@
docker tag openaire/airflow:2.8.2 localhost:5001/airflow:2.8.1
docker push localhost:5001/airflow:2.8.1

69
envs/gcp/airflow.yaml Normal file
View File

@ -0,0 +1,69 @@
#
#
#
useStandardNaming: true
createUserJob:
useHelmHooks: false
applyCustomEnv: false
migrateDatabaseJob:
useHelmHooks: false
applyCustomEnv: false
# Airflow executor
executor: "KubernetesExecutor"
# Secrets for all airflow containers
secret:
# - envName: ""
# secretName: ""
# secretKey: ""
#- envName: "AIRFLOW_CONN_S3"
# secretName: "minio"
# secretKey: "s3connection"
- envName: "AIRFLOW_CONN_S3_CONN"
secretName: "s3-conn-secrets"
secretKey: "AIRFLOW_CONN_S3_CONN"
dags:
persistence:
enabled: true
gitSync:
enabled: true
repo: "https://code-repo.d4science.org/giambattista.bloisi/lot1-kickoff.git"
branch: "airflow"
subPath: "airflow/dags"
config:
webserver:
expose_config: 'True' # by default this is 'False'
#base_url: "http://localhost/"
logging:
remote_logging: "True"
logging_level: "INFO"
remote_base_log_folder: "s3://lot1-airflow/logs"
remote_log_conn_id: "s3_conn"
encrypt_s3_logs: "False"
ingress:
enabled: true
## WARNING: set as "networking.k8s.io/v1beta1" for Kubernetes 1.18 and earlier
apiVersion: networking.k8s.io/v1
## airflow webserver ingress configs
web:
annotations: {}
host: "localhost"
path: "/"
## WARNING: requires Kubernetes 1.18 or later, use "kubernetes.io/ingress.class" annotation for older versions
ingressClassName: "nginx"
## flower ingress configs
flower:
annotations: {}
host: "localhost"
path: "/flower"
## WARNING: requires Kubernetes 1.18 or later, use "kubernetes.io/ingress.class" annotation for older versions
ingressClassName: "nginx"

161
envs/gcp/local-path-storage.yaml Normal file
View File

@ -0,0 +1,161 @@
apiVersion: v1
kind: Namespace
metadata:
name: local-path-storage
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: local-path-provisioner-service-account
namespace: local-path-storage
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: local-path-provisioner-role
namespace: local-path-storage
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "watch", "create", "patch", "update", "delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: local-path-provisioner-role
rules:
- apiGroups: [""]
resources: ["nodes", "persistentvolumeclaims", "configmaps", "pods", "pods/log"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "patch", "update", "delete"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "patch"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: local-path-provisioner-bind
namespace: local-path-storage
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: local-path-provisioner-role
subjects:
- kind: ServiceAccount
name: local-path-provisioner-service-account
namespace: local-path-storage
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: local-path-provisioner-bind
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: local-path-provisioner-role
subjects:
- kind: ServiceAccount
name: local-path-provisioner-service-account
namespace: local-path-storage
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: local-path-provisioner
namespace: local-path-storage
spec:
replicas: 1
selector:
matchLabels:
app: local-path-provisioner
template:
metadata:
labels:
app: local-path-provisioner
spec:
serviceAccountName: local-path-provisioner-service-account
containers:
- name: local-path-provisioner
image: rancher/local-path-provisioner:v0.0.26
imagePullPolicy: IfNotPresent
command:
- local-path-provisioner
- --debug
- start
- --config
- /etc/config/config.json
volumeMounts:
- name: config-volume
mountPath: /etc/config/
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
volumes:
- name: config-volume
configMap:
name: local-path-config
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: local-path
annotations:
storageclass.kubernetes.io/is-default-class: "true"
provisioner: rancher.io/local-path
volumeBindingMode: WaitForFirstConsumer
reclaimPolicy: Delete
---
kind: ConfigMap
apiVersion: v1
metadata:
name: local-path-config
namespace: local-path-storage
data:
config.json: |-
{
"nodePathMap":[
{
"node":"DEFAULT_PATH_FOR_NON_LISTED_NODES",
"paths":["/opt/local-path-provisioner"]
}
]
}
setup: |-
#!/bin/sh
set -eu
mkdir -m 0777 -p "$VOL_DIR"
teardown: |-
#!/bin/sh
set -eu
rm -rf "$VOL_DIR"
helperPod.yaml: |-
apiVersion: v1
kind: Pod
metadata:
name: helper-pod
spec:
priorityClassName: system-node-critical
tolerations:
- key: node.kubernetes.io/disk-pressure
operator: Exists
effect: NoSchedule
containers:
- name: helper-pod
image: busybox
imagePullPolicy: IfNotPresent

22
envs/gcp/nginx.yaml Normal file
View File

@ -0,0 +1,22 @@
apiVersion: helm.cattle.io/v1
kind: HelmChartConfig
metadata:
name: rke2-ingress-nginx
namespace: kube-system
spec:
valuesContent: |-
controller:
ingressClassResource:
controllerValue: "k8s.io/carriershipper-ingress-nginx"
config:
use-forwarded-headers: "true"
enable-real-ip: "true"
proxy-buffer-size: "256k"
proxy-buffer-number: "4"
large-client-header-buffers: "4 16k"
metrics:
enabled: true
serviceMonitor:
enabled: true
additionalLabels:
cluster: nonproduction

63
envs/gcp/opensearch-cluster.yaml Normal file
View File

@ -0,0 +1,63 @@
opensearchCluster:
enabled: true
general:
serviceName: opensearch-cluster
version: 2.12.0
security:
config:
adminCredentialsSecret:
name: admin-credentials-secret # The secret with the admin credentials for the operator to use
securityConfigSecret:
name: securityconfig-secret # The secret containing your customized securityconfig
adminSecret:
name: opensearch-admin-certs
tls:
transport:
generate: false
perNode: false
secret:
name: opensearch-certs
nodesDn: ["CN=Opensearch_Node", ]
adminDn: ["CN=OpenSearch_Admin", ]
http:
generate: false
secret:
name: opensearch-certs
dashboards:
opensearchCredentialsSecret:
name: admin-credentials-secret # This is the name of your secret that contains the credentials for Dashboards to use
enable: true
version: 2.12.0
replicas: 1
resources:
requests:
memory: "512Mi"
cpu: "200m"
limits:
memory: "512Mi"
cpu: "200m"
tls:
enable: true
generate: false
secret:
name: opensearch-dashboards-certs
nodePools:
- component: nodes
replicas: 3
diskSize: "250Gi"
nodeSelector:
jvm: -Xmx12G -Xms12G -XX:ActiveProcessorCount=8
resources:
requests:
memory: "24Gi"
cpu: "4000m"
limits:
memory: "24Gi"
cpu: "8000m"
roles:
- "cluster_manager"
- "data"
persistence:
pvc:
accessModes: # You can change the accessMode
- ReadWriteOnce

22
envs/gcp/opensearch-ingress.yaml Normal file
View File

@ -0,0 +1,22 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: opensearch-ingress
namespace: oa-opensearch
annotations:
# nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
# nginx.ingress.kubernetes.io/rewrite-target: /$1
kubernetes.io/ingress.class: nginx
spec:
ingressClassName: nginx
rules:
- host: "dashboard.opensearch.lot1.xyz"
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: opensearch-cluster-dashboards
port:
number: 5601

13
envs/local/airbyte.yaml Normal file
View File

@ -0,0 +1,13 @@
webapp:
ingress:
enabled: true
className: "nginx"
annotations:
kubernetes.io/ingress.class: nginx
hosts:
- host: localhost
paths:
- path: /
pathType: ImplementationSpecific
tls: []

69
envs/local/airflow.yaml Normal file
View File

@ -0,0 +1,69 @@
#
#
#
useStandardNaming: true
createUserJob:
useHelmHooks: false
applyCustomEnv: false
migrateDatabaseJob:
useHelmHooks: false
applyCustomEnv: false
# Airflow executor
executor: "KubernetesExecutor"
# Secrets for all airflow containers
secret:
# - envName: ""
# secretName: ""
# secretKey: ""
#- envName: "AIRFLOW_CONN_S3"
# secretName: "minio"
# secretKey: "s3connection"
- envName: "AIRFLOW_CONN_S3_CONN"
secretName: "s3-conn-secrets"
secretKey: "AIRFLOW_CONN_S3_CONN"
dags:
persistence:
enabled: true
gitSync:
enabled: true
repo: "https://code-repo.d4science.org/giambattista.bloisi/lot1-kickoff.git"
branch: "airflow"
subPath: "airflow/dags"
config:
webserver:
expose_config: 'True' # by default this is 'False'
#base_url: "http://localhost/"
logging:
remote_logging: "True"
logging_level: "INFO"
remote_base_log_folder: "s3://lot1-airflow/logs"
remote_log_conn_id: "s3_conn"
encrypt_s3_logs: "False"
ingress:
enabled: true
## WARNING: set as "networking.k8s.io/v1beta1" for Kubernetes 1.18 and earlier
apiVersion: networking.k8s.io/v1
## airflow webserver ingress configs
web:
annotations: {}
host: "localhost"
path: "/"
## WARNING: requires Kubernetes 1.18 or later, use "kubernetes.io/ingress.class" annotation for older versions
ingressClassName: "nginx"
## flower ingress configs
flower:
annotations: {}
host: "localhost"
path: "/flower"
## WARNING: requires Kubernetes 1.18 or later, use "kubernetes.io/ingress.class" annotation for older versions
ingressClassName: "nginx"

0
envs/local/common.yaml Normal file
View File

457
envs/local/minio-tenant.yaml Normal file
View File

@ -0,0 +1,457 @@
###
# Root key for dynamically creating a secret for use with configuring root MinIO User
# Specify the ``name`` and then a list of environment variables.
#
# .. important::
#
# Do not use this in production environments.
# This field is intended for use with rapid development or testing only.
#
# For example:
#
# .. code-block:: yaml
#
# name: myminio-env-configuration
# accessKey: minio
# secretKey: minio123
#
secrets:
name: myminio-env-configuration
accessKey: minio
secretKey: minio123
###
# The name of an existing Kubernetes secret to import to the MinIO Tenant
# The secret must contain a key ``config.env``.
# The values should be a series of export statements to set environment variables for the Tenant.
# For example:
#
# .. code-block:: shell
#
# stringData:
# config.env: | -
# export MINIO_ROOT_USER=ROOTUSERNAME
# export MINIO_ROOT_PASSWORD=ROOTUSERPASSWORD
#
#existingSecret:
# name: myminio-env-configuration
###
# Root key for MinIO Tenant Chart
tenant:
###
# The Tenant name
#
# Change this to match your preferred MinIO Tenant name.
name: myminio
###
# Specify the Operator container image to use for the deployment.
# ``image.tag``
# For example, the following sets the image to the ``quay.io/minio/operator`` repo and the v5.0.12 tag.
# The container pulls the image if not already present:
#
# .. code-block:: yaml
#
# image:
# repository: quay.io/minio/minio
# tag: RELEASE.2024-02-09T21-25-16Z
# pullPolicy: IfNotPresent
#
# The chart also supports specifying an image based on digest value:
#
# .. code-block:: yaml
#
# image:
# repository: quay.io/minio/minio@sha256
# digest: 28c80b379c75242c6fe793dfbf212f43c602140a0de5ebe3d9c2a3a7b9f9f983
# pullPolicy: IfNotPresent
#
#
image:
repository: quay.io/minio/minio
tag: RELEASE.2024-02-09T21-25-16Z
pullPolicy: IfNotPresent
###
#
# An array of Kubernetes secrets to use for pulling images from a private ``image.repository``.
# Only one array element is supported at this time.
imagePullSecret: { }
###
# The Kubernetes `Scheduler <https://kubernetes.io/docs/concepts/scheduling-eviction/kube-scheduler/>`__ to use for dispatching Tenant pods.
#
# Specify an empty dictionary ``{}`` to dispatch pods with the default scheduler.
scheduler: { }
###
# The Kubernetes secret name that contains MinIO environment variable configurations.
# The secret is expected to have a key named config.env containing environment variables exports.
configuration:
name: myminio-env-configuration
###
# Top level key for configuring MinIO Pool(s) in this Tenant.
#
# See `Operator CRD: Pools <https://min.io/docs/minio/kubernetes/upstream/reference/operator-crd.html#pool>`__ for more information on all subfields.
pools:
###
# The number of MinIO Tenant Pods / Servers in this pool.
# For standalone mode, supply 1. For distributed mode, supply 4 or more.
# Note that the operator does not support upgrading from standalone to distributed mode.
- servers: 1
###
# Custom name for the pool
name: pool-0
###
# The number of volumes attached per MinIO Tenant Pod / Server.
volumesPerServer: 4
###
# The capacity per volume requested per MinIO Tenant Pod.
size: 1Gi
###
# The `storageClass <https://kubernetes.io/docs/concepts/storage/storage-classes/>`__ to associate with volumes generated for this pool.
#
# If using Amazon Elastic Block Store (EBS) CSI driver
# Please make sure to set xfs for "csi.storage.k8s.io/fstype" parameter under StorageClass.parameters.
# Docs: https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/parameters.md
# storageClassName: standard
###
# Specify `storageAnnotations <https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/>`__ to associate to PVCs.
storageAnnotations: { }
###
# Specify `annotations <https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/>`__ to associate to Tenant pods.
annotations: { }
###
# Specify `labels <https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/>`__ to associate to Tenant pods.
labels: { }
###
#
# An array of `Toleration labels <https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/>`__ to associate to Tenant pods.
#
# These settings determine the distribution of pods across worker nodes.
tolerations: [ ]
###
# Any `Node Selectors <https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/>`__ to apply to Tenant pods.
#
# The Kubernetes scheduler uses these selectors to determine which worker nodes onto which it can deploy Tenant pods.
#
# If no worker nodes match the specified selectors, the Tenant deployment will fail.
nodeSelector: { }
###
#
# The `affinity <https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes-using-node-affinity/>`__ or anti-affinity settings to apply to Tenant pods.
#
# These settings determine the distribution of pods across worker nodes and can help prevent or allow colocating pods onto the same worker nodes.
affinity: { }
###
#
# The `Requests or Limits <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>`__ for resources to associate to Tenant pods.
#
# These settings can control the minimum and maximum resources requested for each pod.
# If no worker nodes can meet the specified requests, the Operator may fail to deploy.
resources: { }
###
# The Kubernetes `SecurityContext <https://kubernetes.io/docs/tasks/configure-pod-container/security-context/>`__ to use for deploying Tenant resources.
#
# You may need to modify these values to meet your cluster's security and access settings.
#
# We recommend disabling recursive permission changes by setting ``fsGroupChangePolicy`` to ``OnRootMismatch`` as those operations can be expensive for certain workloads (e.g. large volumes with many small files).
securityContext:
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
fsGroupChangePolicy: "OnRootMismatch"
runAsNonRoot: true
###
# The Kubernetes `SecurityContext <https://kubernetes.io/docs/tasks/configure-pod-container/security-context/>`__ to use for deploying Tenant containers.
# You may need to modify these values to meet your cluster's security and access settings.
containerSecurityContext:
runAsUser: 1000
runAsGroup: 1000
runAsNonRoot: true
###
#
# An array of `Topology Spread Constraints <https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/>`__ to associate to Operator Console pods.
#
# These settings determine the distribution of pods across worker nodes.
topologySpreadConstraints: [ ]
###
#
# The name of a custom `Container Runtime <https://kubernetes.io/docs/concepts/containers/runtime-class/>`__ to use for the Operator Console pods.
# runtimeClassName: ""
###
# The mount path where Persistent Volumes are mounted inside Tenant container(s).
mountPath: /export
###
# The Sub path inside Mount path where MinIO stores data.
#
# .. warning::
#
# Treat the ``mountPath`` and ``subPath`` values as immutable once you deploy the Tenant.
# If you change these values post-deployment, then you may have different paths for new and pre-existing data.
# This can vastly increase operational complexity and may result in unpredictable data states.
subPath: /data
###
# Configures a Prometheus-compatible scraping endpoint at the specified port.
metrics:
enabled: false
port: 9000
protocol: http
###
# Configures external certificate settings for the Tenant.
certificate:
###
# Specify an array of Kubernetes TLS secrets, where each entry corresponds to a secret the TLS private key and public certificate pair.
#
# This is used by MinIO to verify TLS connections from clients using those CAs
# If you omit this and have clients using TLS certificates minted by an external CA, those connections may fail with warnings around certificate verification.
# See `Operator CRD: TenantSpec <https://min.io/docs/minio/kubernetes/upstream/reference/operator-crd.html#tenantspec>`__.
externalCaCertSecret: [ ]
###
# Specify an array of Kubernetes secrets, where each entry corresponds to a secret contains the TLS private key and public certificate pair.
#
# Omit this to use only the MinIO Operator autogenerated certificates.
#
# If you omit this field *and* set ``requestAutoCert`` to false, the Tenant starts without TLS.
#
# See `Operator CRD: TenantSpec <https://min.io/docs/minio/kubernetes/upstream/reference/operator-crd.html#tenantspec>`__.
#
# .. important::
#
# The MinIO Operator may output TLS connectivity errors if it cannot trust the Certificate Authority (CA) which minted the custom certificates.
#
# You can pass the CA to the Operator to allow it to trust that cert.
# See `Self-Signed, Internal, and Private Certificates <https://min.io/docs/minio/kubernetes/upstream/operations/network-encryption.html#self-signed-internal-and-private-certificates>`__ for more information.
# This step may also be necessary for globally trusted CAs where you must provide intermediate certificates to the Operator to help build the full chain of trust.
externalCertSecret: [ ]
###
# Enable automatic Kubernetes based `certificate generation and signing <https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster>`__
requestAutoCert: true
###
# This field is used only when ``requestAutoCert: true``.
# Use this field to set CommonName for the auto-generated certificate.
# MinIO defaults to using the internal Kubernetes DNS name for the pod
# The default DNS name format is typically ``*.minio.default.svc.cluster.local``.
#
# See `Operator CRD: CertificateConfig <https://min.io/docs/minio/kubernetes/upstream/reference/operator-crd.html#certificateconfig>`__
certConfig: { }
###
# MinIO features to enable or disable in the MinIO Tenant
# See `Operator CRD: Features <https://min.io/docs/minio/kubernetes/upstream/reference/operator-crd.html#features>`__.
features:
bucketDNS: false
domains: { }
enableSFTP: false
###
# Array of objects describing one or more buckets to create during tenant provisioning.
# Example:
#
# .. code-block:: yaml
#
# - name: my-minio-bucket
# objectLock: false # optional
# region: us-east-1 # optional
buckets: [ ]
###
# Array of Kubernetes secrets from which the Operator generates MinIO users during tenant provisioning.
#
# Each secret should specify the ``CONSOLE_ACCESS_KEY`` and ``CONSOLE_SECRET_KEY`` as the access key and secret key for that user.
users: [ ]
###
# The `PodManagement <https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#pod-management-policy>`__ policy for MinIO Tenant Pods.
# Can be "OrderedReady" or "Parallel"
podManagementPolicy: Parallel
# The `Liveness Probe <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes>`__ for monitoring Tenant pod liveness.
# Tenant pods will be restarted if the probe fails.
liveness: { }
###
# `Readiness Probe <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>`__ for monitoring Tenant container readiness.
# Tenant pods will be removed from service endpoints if the probe fails.
readiness: { }
###
# `Startup Probe <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>`__ for monitoring container startup.
# Tenant pods will be restarted if the probe fails.
# Refer
startup: { }
###
# Directs the Operator to deploy the MinIO S3 API and Console services as LoadBalancer objects.
#
# If the Kubernetes cluster has a configured LoadBalancer, it can attempt to route traffic to those services automatically.
#
# - Specify ``minio: true`` to expose the MinIO S3 API.
# - Specify ``console: true`` to expose the Console.
#
# Both fields default to ``false``.
exposeServices: { }
###
# The `Kubernetes Service Account <https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/>`__ associated with the Tenant.
serviceAccountName: ""
###
# Directs the Operator to add the Tenant's metric scrape configuration to an existing Kubernetes Prometheus deployment managed by the Prometheus Operator.
prometheusOperator: false
###
# Configure pod logging configuration for the MinIO Tenant.
#
# - Specify ``json`` for JSON-formatted logs.
# - Specify ``anonymous`` for anonymized logs.
# - Specify ``quiet`` to supress logging.
#
# An example of JSON-formatted logs is as follows:
#
# .. code-block:: shell
#
# $ k logs myminio-pool-0-0 -n default
# {"level":"INFO","errKind":"","time":"2022-04-07T21:49:33.740058549Z","message":"All MinIO sub-systems initialized successfully"}
logging: { }
###
# serviceMetadata allows passing additional labels and annotations to MinIO and Console specific
# services created by the operator.
serviceMetadata: { }
###
# Add environment variables to be set in MinIO container (https://github.com/minio/minio/tree/master/docs/config)
env: [ ]
###
# PriorityClassName indicates the Pod priority and hence importance of a Pod relative to other Pods.
# This is applied to MinIO pods only.
# Refer Kubernetes documentation for details https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass/
priorityClassName: ""
###
# An array of `Volumes <https://kubernetes.io/docs/concepts/storage/volumes/>`__ which the Operator can mount to Tenant pods.
#
# The volumes must exist *and* be accessible to the Tenant pods.
additionalVolumes: [ ]
###
# An array of volume mount points associated to each Tenant container.
#
# Specify each item in the array as follows:
#
# .. code-block:: yaml
#
# volumeMounts:
# - name: volumename
# mountPath: /path/to/mount
#
# The ``name`` field must correspond to an entry in the ``additionalVolumes`` array.
additionalVolumeMounts: [ ]
# Define configuration for KES (stateless and distributed key-management system)
# Refer https://github.com/minio/kes
#kes:
# ## Image field:
# # Image from tag (original behavior), for example:
# # image:
# # repository: quay.io/minio/kes
# # tag: 2024-01-11T13-09-29Z
# # Image from digest (added after original behavior), for example:
# # image:
# # repository: quay.io/minio/kes@sha256
# # digest: fb15af611149892f357a8a99d1bcd8bf5dae713bd64c15e6eb27fbdb88fc208b
# image:
# repository: quay.io/minio/kes
# tag: 2024-01-11T13-09-29Z
# pullPolicy: IfNotPresent
# env: [ ]
# replicas: 2
# configuration: |-
# address: :7373
# tls:
# key: /tmp/kes/server.key # Path to the TLS private key
# cert: /tmp/kes/server.crt # Path to the TLS certificate
# proxy:
# identities: []
# header:
# cert: X-Tls-Client-Cert
# admin:
# identity: ${MINIO_KES_IDENTITY}
# cache:
# expiry:
# any: 5m0s
# unused: 20s
# log:
# error: on
# audit: off
# keystore:
# # KES configured with fs (File System mode) doesn't work in Kubernetes environments and is not recommended
# # use a real KMS
# # fs:
# # path: "./keys" # Path to directory. Keys will be stored as files. Not Recommended for Production.
# vault:
# endpoint: "http://vault.default.svc.cluster.local:8200" # The Vault endpoint
# namespace: "" # An optional Vault namespace. See: https://www.vaultproject.io/docs/enterprise/namespaces/index.html
# prefix: "my-minio" # An optional K/V prefix. The server will store keys under this prefix.
# approle: # AppRole credentials. See: https://www.vaultproject.io/docs/auth/approle.html
# id: "<YOUR APPROLE ID HERE>" # Your AppRole Role ID
# secret: "<YOUR APPROLE SECRET ID HERE>" # Your AppRole Secret ID
# retry: 15s # Duration until the server tries to re-authenticate after connection loss.
# tls: # The Vault client TLS configuration for mTLS authentication and certificate verification
# key: "" # Path to the TLS client private key for mTLS authentication to Vault
# cert: "" # Path to the TLS client certificate for mTLS authentication to Vault
# ca: "" # Path to one or multiple PEM root CA certificates
# status: # Vault status configuration. The server will periodically reach out to Vault to check its status.
# ping: 10s # Duration until the server checks Vault's status again.
# # aws:
# # # The AWS SecretsManager key store. The server will store
# # # secret keys at the AWS SecretsManager encrypted with
# # # AWS-KMS. See: https://aws.amazon.com/secrets-manager
# # secretsmanager:
# # endpoint: "" # The AWS SecretsManager endpoint - e.g.: secretsmanager.us-east-2.amazonaws.com
# # region: "" # The AWS region of the SecretsManager - e.g.: us-east-2
# # kmskey: "" # The AWS-KMS key ID used to en/decrypt secrets at the SecretsManager. By default (if not set) the default AWS-KMS key will be used.
# # credentials: # The AWS credentials for accessing secrets at the AWS SecretsManager.
# # accesskey: "" # Your AWS Access Key
# # secretkey: "" # Your AWS Secret Key
# # token: "" # Your AWS session token (usually optional)
# imagePullPolicy: "IfNotPresent"
# externalCertSecret: null
# clientCertSecret: null
# # Key name to be created on the KMS, default is "my-minio-key"
# keyName: ""
# resources: { }
# nodeSelector: { }
# affinity:
# nodeAffinity: { }
# podAffinity: { }
# podAntiAffinity: { }
# tolerations: [ ]
# annotations: { }
# labels: { }
# serviceAccountName: ""
# securityContext:
# runAsUser: 1000
# runAsGroup: 1000
# runAsNonRoot: true
# fsGroup: 1000
###
# Configures `Ingress <https://kubernetes.io/docs/concepts/services-networking/ingress/>`__ for the Tenant S3 API and Console.
#
# Set the keys to conform to the Ingress controller and configuration of your choice.
ingress:
api:
enabled: true
ingressClassName: "nginx"
labels: { }
annotations:
nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
tls: [ ]
host: minio.local
path: /
pathType: Prefix
console:
enabled: true
ingressClassName: "nginx"
labels: { }
annotations:
nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
tls: [ ]
host: minio-console.local
path: /
pathType: Prefix
# Use an extraResources template section to include additional Kubernetes resources
# with the Helm deployment.
#extraResources:
# - |
# apiVersion: v1
# kind: Secret
# type: Opaque
# metadata:
# name: {{ dig "secrets" "existingSecret" "" (.Values | merge (dict)) }}
# stringData:
# config.env: |-
# export MINIO_ROOT_USER='minio'
# export MINIO_ROOT_PASSWORD='minio123'

57
envs/local/opensearch-cluster.yaml Normal file
View File

@ -0,0 +1,57 @@
opensearchCluster:
enabled: true
general:
serviceName: opensearch-cluster
version: 2.12.0
security:
config:
adminSecret:
name: opensearch-admin-certs
tls:
transport:
generate: false
perNode: false
secret:
name: opensearch-certs
nodesDn: ["CN=Opensearch_Node", ]
adminDn: ["CN=OpenSearch_Admin", ]
http:
generate: false
secret:
name: opensearch-certs
dashboards:
enable: true
version: 2.12.0
replicas: 1
resources:
requests:
memory: "512Mi"
cpu: "200m"
limits:
memory: "512Mi"
cpu: "200m"
tls:
enable: true
generate: false
secret:
name: opensearch-dashboards-certs
nodePools:
- component: nodes
replicas: 3
diskSize: "5Gi"
nodeSelector:
jvm: -Xmx1024M -Xms1024M
resources:
requests:
memory: "2Gi"
cpu: "500m"
limits:
memory: "2Gi"
cpu: "500m"
roles:
- "cluster_manager"
- "data"
persistence:
pvc:
accessModes: # You can change the accessMode
- ReadWriteOnce

6
envs/local/opensearch.yaml Normal file
View File

@ -0,0 +1,6 @@
clusterName: opensearch-cluster
extraEnvs:
- name: DISABLE_INSTALL_DEMO_CONFIG
value: "true"

11
gcp.tfvars Normal file
View File

@ -0,0 +1,11 @@
env = "gcp"
kube_context= "rke2-cluster-0"
domain = "openaire.duckdns.org"
admin_user = "admin"
admin_password = "admin"
admin_hash = "$2y$10$Wd.mnnrDG01KJ42aVtC89.FdXOvyRm4RNfDfZ5F8k4r/fmSZgrIEq" # generate with htpasswd -bnBC 10 "" <admin_password>
s3_endpoint = "https://storage.googleapis.com"
s3_key= "google key"
s3_secret = "google secret"
# bucket skgif-openaire-eu

23
local.tfvars Normal file
View File

@ -0,0 +1,23 @@
env = "local"
kube_context= "kind-local-dataplatform"
domain = "local-dataplatform"
admin_user = "admin"
admin_password = "admin"
admin_hash = "$2y$10$Wd.mnnrDG01KJ42aVtC89.FdXOvyRm4RNfDfZ5F8k4r/fmSZgrIEq" # generate with htpasswd -bnBC 10 "" <admin_password>
s3_endpoint = "https://minio.lot1-minio-tenant.svc.cluster.local"
s3_key= "minio"
s3_secret = "minio123"
/*
{
"type": "s3",
"settings": {
"bucket": "opensearch-repo",
"base_path": "lot1",
"endpoint": "https://minio.lot1-minio-tenant.svc.cluster.local",
"access_key": "minio",
"secret_key": "minio123"
}
}
*/

28
main.tf Normal file
View File

@ -0,0 +1,28 @@
/*module "minio" {
source = "./modules/minio"
kube_context = "kind-openaire-data-platform"
}*/
module "opensearch-cluster" {
source = "./modules/opensearch"
kube_context = var.kube_context
admin_user = var.admin_user
admin_password = var.admin_password
admin_hash = var.admin_hash
env = var.env
domain = var.domain
}
module "airflow" {
source = "./modules/airflow"
kube_context = var.kube_context
admin_user = var.admin_user
admin_password = var.admin_password
admin_hash = var.admin_hash
env = var.env
domain = var.domain
s3_endpoint = var.s3_endpoint
s3_key = var.s3_key
s3_secret = var.s3_secret
}

211
modules/airflow/airflow.tf Normal file
View File

@ -0,0 +1,211 @@
resource "kubernetes_namespace" "spark_jobs_namespace" {
metadata {
name = "${var.namespace_prefix}spark-jobs"
}
}
resource "kubernetes_service_account_v1" "spark_sa" {
metadata {
name = "spark"
namespace = "${var.namespace_prefix}spark-jobs"
}
}
resource "kubernetes_role" "airflow_spark_role" {
metadata {
name = "airflow-spark-role"
namespace = "${var.namespace_prefix}spark-jobs"
}
rule {
api_groups = ["sparkoperator.k8s.io"]
resources = ["sparkapplications", "sparkapplications/status",
"scheduledsparkapplications", "scheduledsparkapplications/status"]
verbs = ["*"]
}
rule {
api_groups = [""]
resources = ["pods/log"]
verbs = ["*"]
}
}
resource "kubernetes_role_binding_v1" "airflow_spark_role_binding" {
metadata {
name = "airflow-spark-role-binding"
namespace = "${var.namespace_prefix}spark-jobs"
}
subject {
kind = "ServiceAccount"
name = "airflow-worker"
namespace = "${var.namespace_prefix}airflow"
}
role_ref {
api_group = "rbac.authorization.k8s.io"
kind = "Role"
name = "airflow-spark-role"
}
}
resource "kubernetes_role_binding_v1" "airflow_spark_role_binding2" {
metadata {
name = "airflow-spark-role-binding2"
namespace = "${var.namespace_prefix}spark-jobs"
}
subject {
kind = "ServiceAccount"
name = "airflow-worker"
namespace = "${var.namespace_prefix}airflow"
}
role_ref {
api_group = "rbac.authorization.k8s.io"
kind = "Role"
name = "spark-role"
}
}
resource "kubernetes_role_binding_v1" "spark_role_binding" {
metadata {
name = "spark-role-binding"
namespace = "${var.namespace_prefix}spark-jobs"
}
subject {
kind = "ServiceAccount"
name = "spark"
namespace = "${var.namespace_prefix}spark-jobs"
}
role_ref {
api_group = "rbac.authorization.k8s.io"
kind = "Role"
name = "spark-role"
}
}
resource "helm_release" "gcp_spark_operator" {
depends_on = [kubernetes_namespace.spark_jobs_namespace]
name = "gcp-spark-operator"
chart = "spark-operator"
repository = "https://kubeflow.github.io/spark-operator"
create_namespace = "true"
namespace = "${var.namespace_prefix}gcp-spark-operator"
dependency_update = "true"
version = "1.1.27"
set {
name = "sparkJobNamespace"
value = "${var.namespace_prefix}spark-jobs"
}
set {
name = "enableWebhook"
value = "true"
}
set {
name = "ingressUrlFormat"
value = "\\{\\{$appName\\}\\}.\\{\\{$appNamespace\\}\\}.${var.domain}"
type = "string"
}
}
resource "kubernetes_namespace" "airflow" {
metadata {
name = "${var.namespace_prefix}airflow"
}
}
resource "kubernetes_secret" "s3_conn_secrets" {
depends_on = [kubernetes_namespace.airflow]
metadata {
name = "s3-conn-secrets"
namespace = "${var.namespace_prefix}airflow"
}
data = {
username = var.s3_key
password = var.s3_secret
AIRFLOW_CONN_S3_CONN = <<EOT
{
"conn_type": "aws",
"extra": {
"aws_access_key_id": "${var.s3_key}",
"aws_secret_access_key": "${var.s3_secret}",
"endpoint_url": "${var.s3_endpoint}",
"verify": false
}
}
EOT
}
type = "Opaque"
}
resource "helm_release" "airflow" {
depends_on = [kubernetes_secret.s3_conn_secrets]
name = "airflow"
chart = "airflow"
repository = "https://airflow.apache.org"
namespace = "${var.namespace_prefix}airflow"
dependency_update = "true"
version = "1.13.0"
values = [
file("./envs/${var.env}/airflow.yaml")
]
set {
name = "fernetkey"
value = "TG9mVjJvVEpoREVYdmdTRWlHdENXQ05zOU5OU2VGY0U="
}
set {
name = "webserver.defaultUser.password"
value = var.admin_password
}
set {
name = "spec.values.env"
value = yamlencode([
{
name = "AIRFLOW__WEBSERVER__BASE_URL",
value = "https://airflow.${var.domain}"
},
{
name = "AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX",
value = "True"
}
])
}
set {
name = "images.airflow.repository"
value = "gbloisi/airflow"
}
set {
name = "images.airflow.tag"
value = "2.8.3rc1-python3.11"
}
set {
name = "ingress.web.host"
value = "airflow.${var.domain}"
}
set {
name = "ingress.flower.host"
value = "airflow.${var.domain}"
}
}

12
modules/airflow/providers.tf Normal file
View File

@ -0,0 +1,12 @@
provider "helm" {
# Several Kubernetes authentication methods are possible: https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs#authentication
kubernetes {
config_path = pathexpand(var.kube_config)
config_context = var.kube_context
}
}
provider "kubernetes" {
config_path = pathexpand(var.kube_config)
config_context = var.kube_context
}

51
modules/airflow/variables.tf Normal file
View File

@ -0,0 +1,51 @@
variable "env" {
type = string
default = "local"
}
variable "kube_config" {
type = string
default = "~/.kube/config"
}
variable "kube_context" {
type = string
default = "default"
}
variable "namespace_prefix" {
type = string
default = "lot1-"
}
variable "domain" {
type = string
default = "local-dataplatform"
}
variable "s3_endpoint" {
type = string
default = "https://minio.lot1-minio-tenant.svc.cluster.local"
}
variable "s3_key" {
type = string
default = "minio"
}
variable "s3_secret" {
type = string
default = "minio123"
}
variable "admin_user" {
type = string
}
variable "admin_password" {
type = string
}
variable "admin_hash" {
type = string
}

34
modules/minio/minio-bucket.yaml Normal file
View File

@ -0,0 +1,34 @@
apiVersion: batch/v1
kind: Job
metadata:
name: create-bucket
namespace: block-storage
spec:
template:
spec:
containers:
- name: createbucket
image: amazon/aws-cli
command: ["aws"]
args:
- s3api
- create-bucket
- --bucket
- postgres
- --endpoint-url
- http://minio:80
env:
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: minio-secret
key: accesskey
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: minio-secret
key: secretkey
restartPolicy: Never
backoffLimit: 1

9
modules/minio/minio-operator.tf Normal file
View File

@ -0,0 +1,9 @@
resource "helm_release" "minio_operator" {
name = "minio-operator"
chart = "operator"
repository = "https://operator.min.io/"
create_namespace = "true"
namespace = "minio-operator"
dependency_update = "true"
version = "5.0.12"
}

59
modules/minio/minio-tenant.tf Normal file
View File

@ -0,0 +1,59 @@
resource "helm_release" "minio_tenant" {
name = "minio-tenant"
chart = "tenant"
repository = "https://operator.min.io/"
create_namespace = "true"
namespace = "${var.namespace_prefix}minio-tenant"
dependency_update = "true"
version = "5.0.12"
values = [
file("./envs/${var.env}/minio-tenant.yaml")
]
set {
name = "ingress.api.host"
value = "minio.${var.domain}"
}
set {
name = "ingress.console.host"
value = "console-minio.${var.domain}"
}
}
/*
resource "kubernetes_manifest" "minio_ingress" {
manifest = yamldecode(<<YAML
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-minio
namespace: block-storage
annotations:
kubernetes.io/ingress.class: "nginx"
## Remove if using CA signed certificate
nginx.ingress.kubernetes.io/proxy-ssl-verify: "off"
nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
nginx.ingress.kubernetes.io/rewrite-target: /
nginx.ingress.kubernetes.io/proxy-body-size: "0"
spec:
ingressClassName: nginx
tls:
- hosts:
- minio.${var.domain}
secretName: nginx-tls
rules:
- host: minio.${var.domain}
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: minio
port:
number: 443
YAML
)
}*/

12
modules/minio/providers.tf Normal file
View File

@ -0,0 +1,12 @@
provider "helm" {
# Several Kubernetes authentication methods are possible: https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs#authentication
kubernetes {
config_path = pathexpand(var.kube_config)
config_context = var.kube_context
}
}
provider "kubernetes" {
config_path = pathexpand(var.kube_config)
config_context = var.kube_context
}

24
modules/minio/variables.tf Normal file
View File

@ -0,0 +1,24 @@
variable "env" {
type = string
default = "local"
}
variable "kube_config" {
type = string
default = "~/.kube/config"
}
variable "kube_context" {
type = string
default = "default"
}
variable "namespace_prefix" {
type = string
default = "lot1-"
}
variable "domain" {
type = string
default = "local-dataplatform"
}

152
modules/opensearch/opensearch-certificates.tf Normal file
View File

@ -0,0 +1,152 @@
resource "kubernetes_manifest" "opensearch_issuer" {
depends_on = [kubernetes_namespace.opensearch_cluster]
manifest = yamldecode(<<YAML
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
name: selfsigned-issuer
namespace: "${var.namespace_prefix}opensearch-cluster"
spec:
selfSigned: {}
YAML
)
}
resource "kubernetes_manifest" "opensearch_ca_certificate" {
depends_on = [kubernetes_namespace.opensearch_cluster]
manifest = yamldecode(<<YAML
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: ca-certificate
namespace: "${var.namespace_prefix}opensearch-cluster"
spec:
secretName: ca-cert
duration: 9000h # ~1year
renewBefore: 360h # 15d
commonName: Test CA
isCA: true
privateKey:
size: 2048
usages:
- digital signature
- key encipherment
issuerRef:
name: selfsigned-issuer
YAML
)
}
resource "kubernetes_manifest" "opensearch_ca_issuer" {
depends_on = [kubernetes_namespace.opensearch_cluster]
manifest = yamldecode(<<YAML
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
name: ca-issuer
namespace: "${var.namespace_prefix}opensearch-cluster"
spec:
ca:
secretName: ca-cert
YAML
)
}
resource "kubernetes_manifest" "opensearch_cluster_certificate" {
depends_on = [kubernetes_namespace.opensearch_cluster]
manifest = yamldecode(<<YAML
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: opensearch-certs
namespace: "${var.namespace_prefix}opensearch-cluster"
spec:
secretName: opensearch-certs
duration: 9000h # ~1year
renewBefore: 360h # 15d
isCA: false
privateKey:
size: 2048
algorithm: RSA
encoding: PKCS8
dnsNames:
- opensearch-cluster.${var.domain}
- opensearch-cluster
- opensearch-cluster-masters-0
- opensearch-cluster-masters-1
- opensearch-cluster-masters-2
- opensearch-cluster-bootstrap-0
usages:
- signing
- key encipherment
- server auth
- client auth
commonName: Opensearch_Node
issuerRef:
name: ca-issuer
YAML
)
}
resource "kubernetes_manifest" "opensearch_admin_certificate" {
depends_on = [kubernetes_namespace.opensearch_cluster]
manifest = yamldecode(<<YAML
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: opensearch-admin-certs
namespace: "${var.namespace_prefix}opensearch-cluster"
spec:
secretName: opensearch-admin-certs
duration: 9000h # ~1year
renewBefore: 360h # 15d
isCA: false
privateKey:
size: 2048
algorithm: RSA
encoding: PKCS8
commonName: OpenSearch_Admin
usages:
- signing
- key encipherment
- server auth
- client auth
issuerRef:
name: ca-issuer
YAML
)
}
resource "kubernetes_manifest" "opensearch_dashboard_certificate" {
depends_on = [kubernetes_namespace.opensearch_cluster]
manifest = yamldecode(<<YAML
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: opensearch-dashboards-certs
namespace: "${var.namespace_prefix}opensearch-cluster"
spec:
secretName: opensearch-dashboards-certs
duration: 9000h # ~1year
renewBefore: 360h # 15d
isCA: false
privateKey:
size: 2048
algorithm: RSA
encoding: PKCS8
dnsNames:
- opensearch-cluster-dashboards
usages:
- signing
- key encipherment
- server auth
- client auth
issuerRef:
name: ca-issuer
YAML
)
}

358
modules/opensearch/opensearch-cluster.tf Normal file
View File

@ -0,0 +1,358 @@
resource "kubernetes_manifest" "opensearch_securityconfig_secret" {
depends_on = [kubernetes_namespace.opensearch_cluster]
computed_fields = ["stringData"]
manifest = yamldecode(<<YAML
apiVersion: v1
kind: Secret
metadata:
name: securityconfig-secret
namespace: "${var.namespace_prefix}opensearch-cluster"
type: Opaque
stringData:
action_groups.yml: |-
_meta:
type: "actiongroups"
config_version: 2
internal_users.yml: |-
_meta:
type: "internalusers"
config_version: 2
admin:
hash: "${var.admin_hash}"
reserved: true
backend_roles:
- "admin"
description: "Demo admin user"
dashboarduser:
hash: "${var.admin_hash}"
reserved: true
description: "Demo OpenSearch Dashboards user"
nodes_dn.yml: |-
_meta:
type: "nodesdn"
config_version: 2
whitelist.yml: |-
_meta:
type: "whitelist"
config_version: 2
tenants.yml: |-
_meta:
type: "tenants"
config_version: 2
roles_mapping.yml: |-
_meta:
type: "rolesmapping"
config_version: 2
all_access:
reserved: false
backend_roles:
- "admin"
description: "Maps admin to all_access"
own_index:
reserved: false
users:
- "*"
description: "Allow full access to an index named like the username"
readall:
reserved: false
backend_roles:
- "readall"
manage_snapshots:
reserved: false
backend_roles:
- "snapshotrestore"
dashboard_server:
reserved: true
users:
- "dashboarduser"
roles.yml: |-
_meta:
type: "roles"
config_version: 2
dashboard_read_only:
reserved: true
security_rest_api_access:
reserved: true
# Allows users to view monitors, destinations and alerts
alerting_read_access:
reserved: true
cluster_permissions:
- 'cluster:admin/opendistro/alerting/alerts/get'
- 'cluster:admin/opendistro/alerting/destination/get'
- 'cluster:admin/opendistro/alerting/monitor/get'
- 'cluster:admin/opendistro/alerting/monitor/search'
# Allows users to view and acknowledge alerts
alerting_ack_alerts:
reserved: true
cluster_permissions:
- 'cluster:admin/opendistro/alerting/alerts/*'
# Allows users to use all alerting functionality
alerting_full_access:
reserved: true
cluster_permissions:
- 'cluster_monitor'
- 'cluster:admin/opendistro/alerting/*'
index_permissions:
- index_patterns:
- '*'
allowed_actions:
- 'indices_monitor'
- 'indices:admin/aliases/get'
- 'indices:admin/mappings/get'
# Allow users to read Anomaly Detection detectors and results
anomaly_read_access:
reserved: true
cluster_permissions:
- 'cluster:admin/opendistro/ad/detector/info'
- 'cluster:admin/opendistro/ad/detector/search'
- 'cluster:admin/opendistro/ad/detectors/get'
- 'cluster:admin/opendistro/ad/result/search'
- 'cluster:admin/opendistro/ad/tasks/search'
- 'cluster:admin/opendistro/ad/detector/validate'
- 'cluster:admin/opendistro/ad/result/topAnomalies'
# Allows users to use all Anomaly Detection functionality
anomaly_full_access:
reserved: true
cluster_permissions:
- 'cluster_monitor'
- 'cluster:admin/opendistro/ad/*'
index_permissions:
- index_patterns:
- '*'
allowed_actions:
- 'indices_monitor'
- 'indices:admin/aliases/get'
- 'indices:admin/mappings/get'
# Allows users to read Notebooks
notebooks_read_access:
reserved: true
cluster_permissions:
- 'cluster:admin/opendistro/notebooks/list'
- 'cluster:admin/opendistro/notebooks/get'
# Allows users to all Notebooks functionality
notebooks_full_access:
reserved: true
cluster_permissions:
- 'cluster:admin/opendistro/notebooks/create'
- 'cluster:admin/opendistro/notebooks/update'
- 'cluster:admin/opendistro/notebooks/delete'
- 'cluster:admin/opendistro/notebooks/get'
- 'cluster:admin/opendistro/notebooks/list'
# Allows users to read observability objects
observability_read_access:
reserved: true
cluster_permissions:
- 'cluster:admin/opensearch/observability/get'
# Allows users to all Observability functionality
observability_full_access:
reserved: true
cluster_permissions:
- 'cluster:admin/opensearch/observability/create'
- 'cluster:admin/opensearch/observability/update'
- 'cluster:admin/opensearch/observability/delete'
- 'cluster:admin/opensearch/observability/get'
# Allows users to read and download Reports
reports_instances_read_access:
reserved: true
cluster_permissions:
- 'cluster:admin/opendistro/reports/instance/list'
- 'cluster:admin/opendistro/reports/instance/get'
- 'cluster:admin/opendistro/reports/menu/download'
# Allows users to read and download Reports and Report-definitions
reports_read_access:
reserved: true
cluster_permissions:
- 'cluster:admin/opendistro/reports/definition/get'
- 'cluster:admin/opendistro/reports/definition/list'
- 'cluster:admin/opendistro/reports/instance/list'
- 'cluster:admin/opendistro/reports/instance/get'
- 'cluster:admin/opendistro/reports/menu/download'
# Allows users to all Reports functionality
reports_full_access:
reserved: true
cluster_permissions:
- 'cluster:admin/opendistro/reports/definition/create'
- 'cluster:admin/opendistro/reports/definition/update'
- 'cluster:admin/opendistro/reports/definition/on_demand'
- 'cluster:admin/opendistro/reports/definition/delete'
- 'cluster:admin/opendistro/reports/definition/get'
- 'cluster:admin/opendistro/reports/definition/list'
- 'cluster:admin/opendistro/reports/instance/list'
- 'cluster:admin/opendistro/reports/instance/get'
- 'cluster:admin/opendistro/reports/menu/download'
# Allows users to use all asynchronous-search functionality
asynchronous_search_full_access:
reserved: true
cluster_permissions:
- 'cluster:admin/opendistro/asynchronous_search/*'
index_permissions:
- index_patterns:
- '*'
allowed_actions:
- 'indices:data/read/search*'
# Allows users to read stored asynchronous-search results
asynchronous_search_read_access:
reserved: true
cluster_permissions:
- 'cluster:admin/opendistro/asynchronous_search/get'
# Allows user to use all index_management actions - ism policies, rollups, transforms
index_management_full_access:
reserved: true
cluster_permissions:
- "cluster:admin/opendistro/ism/*"
- "cluster:admin/opendistro/rollup/*"
- "cluster:admin/opendistro/transform/*"
index_permissions:
- index_patterns:
- '*'
allowed_actions:
- 'indices:admin/opensearch/ism/*'
# Allows users to use all cross cluster replication functionality at leader cluster
cross_cluster_replication_leader_full_access:
reserved: true
index_permissions:
- index_patterns:
- '*'
allowed_actions:
- "indices:admin/plugins/replication/index/setup/validate"
- "indices:data/read/plugins/replication/changes"
- "indices:data/read/plugins/replication/file_chunk"
# Allows users to use all cross cluster replication functionality at follower cluster
cross_cluster_replication_follower_full_access:
reserved: true
cluster_permissions:
- "cluster:admin/plugins/replication/autofollow/update"
index_permissions:
- index_patterns:
- '*'
allowed_actions:
- "indices:admin/plugins/replication/index/setup/validate"
- "indices:data/write/plugins/replication/changes"
- "indices:admin/plugins/replication/index/start"
- "indices:admin/plugins/replication/index/pause"
- "indices:admin/plugins/replication/index/resume"
- "indices:admin/plugins/replication/index/stop"
- "indices:admin/plugins/replication/index/update"
- "indices:admin/plugins/replication/index/status_check"
config.yml: |-
_meta:
type: "config"
config_version: "2"
config:
dynamic:
http:
anonymous_auth_enabled: false
authc:
basic_internal_auth_domain:
http_enabled: true
transport_enabled: true
order: "4"
http_authenticator:
type: basic
challenge: true
authentication_backend:
type: intern
YAML
)
}
resource "kubernetes_secret" "opensearch_admin_credential_secrets" {
depends_on = [kubernetes_namespace.opensearch_cluster]
metadata {
name = "admin-credentials-secret"
namespace = "${var.namespace_prefix}opensearch-cluster"
}
data = {
username = "admin"
password = var.admin_password
}
type = "Opaque"
}
resource "helm_release" "opensearch-cluster" {
depends_on = [helm_release.opensearch-operator, kubernetes_namespace.opensearch_cluster, kubernetes_manifest.opensearch_cluster_certificate, kubernetes_manifest.opensearch_dashboard_certificate]
chart = "opensearch-cluster"
name = "opensearch-cluster"
namespace = "${var.namespace_prefix}opensearch-cluster"
create_namespace = false
repository = "https://opensearch-project.github.io/opensearch-k8s-operator/"
version = "2.5.1"
values = [
file("./envs/${var.env}/opensearch-cluster.yaml")
]
}
resource "kubernetes_manifest" "opensearch_dashboard_ingress" {
depends_on = [helm_release.opensearch-cluster]
manifest = yamldecode(<<YAML
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: opensearch-dashboard-ingress
namespace: "${var.namespace_prefix}opensearch-cluster"
annotations:
kubernetes.io/ingress.class: nginx
cert-manager.io/cluster-issuer: cert-manager-webhook-duckdns-staging
spec:
ingressClassName: nginx
tls:
- hosts:
- "opensearch-cluster-dashboards.${var.domain}"
secretName: "opensearch-cluster-dashboards-tls-secret-staging"
rules:
- host: "opensearch-cluster-dashboards.${var.domain}"
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: opensearch-cluster-dashboards
port:
number: 5601
YAML
)
}
resource "kubernetes_manifest" "opensearch_ingress" {
depends_on = [helm_release.opensearch-cluster]
manifest = yamldecode(<<YAML
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: opensearch-ingress
namespace: "${var.namespace_prefix}opensearch-cluster"
annotations:
nginx.ingress.kubernetes.io/proxy-ssl-verify: "false"
nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
kubernetes.io/ingress.class: nginx
cert-manager.io/cluster-issuer: cert-manager-webhook-duckdns-staging
spec:
ingressClassName: nginx
tls:
- hosts:
- "opensearch-cluster.${var.domain}"
secretName: "opensearch-cluster-tls-secret-staging"
rules:
- host: "opensearch-cluster.${var.domain}"
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: opensearch-cluster
port:
number: 9200
YAML
)
}

37
modules/opensearch/opensearch-operator.tf Normal file
View File

@ -0,0 +1,37 @@
resource "kubernetes_namespace" "opensearch_operator" {
metadata {
name = "${var.namespace_prefix}opensearch-operator"
}
}
resource "kubernetes_namespace" "opensearch_cluster" {
metadata {
name = "${var.namespace_prefix}opensearch-cluster"
}
}
resource "helm_release" "opensearch-operator" {
depends_on = [kubernetes_namespace.opensearch_operator, kubernetes_namespace.opensearch_cluster]
chart = "opensearch-operator"
name = "opensearch-operator"
namespace = "${var.namespace_prefix}opensearch-operator"
create_namespace = false
repository = "https://opensearch-project.github.io/opensearch-k8s-operator/"
version = "2.5.1"
set {
name = "manager.watchNamespace"
value = "${var.namespace_prefix}opensearch-cluster"
}
# You can provide a map of value using yamldecode. Don't forget to escape the last element after point in the name
set {
name = "manager\\.extraEnv"
value = yamlencode({
name = "SKIP_INIT_CONTAINER",
value = "true"
})
}
}

24
modules/opensearch/providers.tf Normal file
View File

@ -0,0 +1,24 @@
terraform {
required_providers {
bcrypt = {
source = "viktorradnai/bcrypt"
version = "0.1.2"
}
}
}
provider "bcrypt" {
}
provider "helm" {
# Several Kubernetes authentication methods are possible: https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs#authentication
kubernetes {
config_path = pathexpand(var.kube_config)
config_context = var.kube_context
}
}
provider "kubernetes" {
config_path = pathexpand(var.kube_config)
config_context = var.kube_context
}

37
modules/opensearch/variables.tf Normal file
View File

@ -0,0 +1,37 @@
variable "env" {
type = string
default = "local"
}
variable "kube_config" {
type = string
default = "~/.kube/config"
}
variable "kube_context" {
type = string
default = "default"
}
variable "namespace_prefix" {
type = string
default = "lot1-"
}
variable "domain" {
type = string
default = "local-dataplatform"
}
variable "admin_user" {
type = string
}
variable "admin_password" {
type = string
}
variable "admin_hash" {
type = string
}

46
variables.tf Normal file
View File

@ -0,0 +1,46 @@
variable "env" {
type = string
default = "local"
}
variable "kube_config" {
type = string
default = "~/.kube/config"
}
variable "kube_context" {
type = string
default = "default"
}
variable "namespace_prefix" {
type = string
default = "lot1-"
}
variable "domain" {
type = string
default = "local-dataplatform"
}
variable "admin_user" {
type = string
}
variable "admin_password" {
type = string
}
variable "admin_hash" {
type = string
}
variable "s3_endpoint" {
default = "https://minio.lot1-minio-tenant.svc.cluster.local"
}
variable "s3_key" {
default = "minio"
}
variable "s3_secret" {
default = "minio123"
}