envs/local/airflow.yaml

@@ -31,9 +31,9 @@ dags:
     enabled: true
   gitSync:
     enabled: true
-    repo: "https://code-repo.d4science.org/D-Net/code-infrasturcutre-lab.git"
+    repo: "https://code-repo.d4science.org/D-Net/code-infrastructure-lab.git"
-    branch: "airflow"
+    branch: "master"
-    subPath: "airflow/dags"
+    subPath: "workflow/dnet"
 
 config:
   webserver:
@@ -42,7 +42,7 @@ config:
   logging:
     remote_logging: "True"
     logging_level: "INFO"
-    remote_base_log_folder: "s3://dnet-airflow/logs"
+    remote_base_log_folder: "s3://workflow-logs/logs"
     remote_log_conn_id: "s3_conn"
     encrypt_s3_logs: "False"
 

envs/local/minio-tenant.yaml

@@ -1,42 +1,3 @@
-
-
-###
-# Root key for dynamically creating a secret for use with configuring root MinIO User
-# Specify the ``name`` and then a list of environment variables.
-#
-# .. important::
-#
-#    Do not use this in production environments.
-#    This field is intended for use with rapid development or testing only.
-#
-# For example:
-#
-# .. code-block:: yaml
-#
-#    name: myminio-env-configuration
-#    accessKey: minio
-#    secretKey: minio123
-#
-secrets:
-  name: myminio-env-configuration
-  accessKey: minio
-  secretKey: minio123
-  ###
-  # The name of an existing Kubernetes secret to import to the MinIO Tenant
-  # The secret must contain a key ``config.env``.
-  # The values should be a series of export statements to set environment variables for the Tenant.
-  # For example:
-  #
-  # .. code-block:: shell
-  #
-  #    stringData:
-  #       config.env: | -
-  #         export MINIO_ROOT_USER=ROOTUSERNAME
-  #         export MINIO_ROOT_PASSWORD=ROOTUSERPASSWORD
-  #
-  #existingSecret:
-  #  name: myminio-env-configuration
-###
 # Root key for MinIO Tenant Chart
 tenant:
   ###
@@ -47,14 +8,14 @@ tenant:
   ###
   # Specify the Operator container image to use for the deployment.
   # ``image.tag``
-  # For example, the following sets the image to the ``quay.io/minio/operator`` repo and the v5.0.12 tag.
+  # For example, the following sets the image to the ``quay.io/minio/operator`` repo and the v6.0.4 tag.
   # The container pulls the image if not already present:
   #
   # .. code-block:: yaml
   #
   #    image:
   #       repository: quay.io/minio/minio
-  #       tag: RELEASE.2024-02-09T21-25-16Z
+  #       tag: RELEASE.2024-10-02T17-50-41Z
   #       pullPolicy: IfNotPresent
   #
   # The chart also supports specifying an image based on digest value:
@@ -69,7 +30,7 @@ tenant:
   #
   image:
     repository: quay.io/minio/minio
-    tag: RELEASE.2024-02-09T21-25-16Z
+    tag: RELEASE.2024-10-02T17-50-41Z
     pullPolicy: IfNotPresent
   ###
   #
@@ -87,6 +48,44 @@ tenant:
   configuration:
     name: myminio-env-configuration
   ###
+  # Root key for dynamically creating a secret for use with configuring root MinIO User
+  # Specify the ``name`` and then a list of environment variables.
+  #
+  # .. important::
+  #
+  #    Do not use this in production environments.
+  #    This field is intended for use with rapid development or testing only.
+  #
+  # For example:
+  #
+  # .. code-block:: yaml
+  #
+  #    name: myminio-env-configuration
+  #    accessKey: minio
+  #    secretKey: minio123
+  #
+  configSecret:
+    name: myminio-env-configuration
+    accessKey: minio
+    secretKey: minio123
+    #existingSecret: true
+
+  ###
+  # If this variable is set to true, then enable the usage of an existing Kubernetes secret to set environment variables for the Tenant.
+  # The existing Kubernetes secret name must be placed under .tenant.configuration.name e.g. existing-minio-env-configuration
+  # The secret must contain a key ``config.env``.
+  # The values should be a series of export statements to set environment variables for the Tenant.
+  # For example:
+  #
+  # .. code-block:: shell
+  #
+  #    stringData:
+  #       config.env: |-
+  #         export MINIO_ROOT_USER=ROOTUSERNAME
+  #         export MINIO_ROOT_PASSWORD=ROOTUSERPASSWORD
+  #
+  #   existingSecret: false
+  ###
   # Top level key for configuring MinIO Pool(s) in this Tenant.
   #
   # See `Operator CRD: Pools <https://min.io/docs/minio/kubernetes/upstream/reference/operator-crd.html#pool>`__ for more information on all subfields.
@@ -104,7 +103,7 @@ tenant:
       volumesPerServer: 4
       ###
       # The capacity per volume requested per MinIO Tenant Pod.
-      size: 1Gi
+      size: 50Gi
       ###
       # The `storageClass <https://kubernetes.io/docs/concepts/storage/storage-classes/>`__ to associate with volumes generated for this pool.
       #
@@ -166,6 +165,12 @@ tenant:
         runAsUser: 1000
         runAsGroup: 1000
         runAsNonRoot: true
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop:
+            - ALL
+        seccompProfile:
+          type: RuntimeDefault
       ###
       #
       # An array of `Topology Spread Constraints <https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/>`__ to associate to Operator Console pods.
@@ -225,6 +230,10 @@ tenant:
     # Enable automatic Kubernetes based `certificate generation and signing <https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster>`__
     requestAutoCert: true
     ###
+    # The minimum number of days to expiry before an alert for an expiring certificate is fired.
+    # In the below example, if a given certificate will expire in 7 days then expiration events will only be triggered 1 day before expiry
+    # certExpiryAlertThreshold: 1
+    ###
     # This field is used only when ``requestAutoCert: true``.
     # Use this field to set CommonName for the auto-generated certificate.
     # MinIO defaults to using the internal Kubernetes DNS name for the pod
@@ -248,7 +257,7 @@ tenant:
   #    - name: my-minio-bucket
   #         objectLock: false        # optional
   #         region: us-east-1        # optional
-  buckets: [ ]
+  buckets: [ "workflow-logs", "binaries", "graph"]
   ###
   # Array of Kubernetes secrets from which the Operator generates MinIO users during tenant provisioning.
   #
@@ -271,6 +280,9 @@ tenant:
   # Refer
   startup: { }
   ###
+  # The `Lifecycle hooks <https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/>`__ for container.
+  lifecycle: { }
+  ###
   # Directs the Operator to deploy the MinIO S3 API and Console services as LoadBalancer objects.
   #
   # If the Kubernetes cluster has a configured LoadBalancer, it can attempt to route traffic to those services automatically.
@@ -337,14 +349,14 @@ tenant:
   #  # Image from tag (original behavior), for example:
   #  # image:
   #  #   repository: quay.io/minio/kes
-  #  #   tag: 2024-01-11T13-09-29Z
+  #  #   tag: 2024-09-11T07-22-50Z
   #  # Image from digest (added after original behavior), for example:
   #  # image:
   #  #   repository: quay.io/minio/kes@sha256
   #  #   digest: fb15af611149892f357a8a99d1bcd8bf5dae713bd64c15e6eb27fbdb88fc208b
   #  image:
   #    repository: quay.io/minio/kes
-  #    tag: 2024-01-11T13-09-29Z
+  #    tag: 2024-09-11T07-22-50Z
   #    pullPolicy: IfNotPresent
   #  env: [ ]
   #  replicas: 2
@@ -417,6 +429,17 @@ tenant:
   #    runAsGroup: 1000
   #    runAsNonRoot: true
   #    fsGroup: 1000
+  #  containerSecurityContext:
+  #    runAsUser: 1000
+  #    runAsGroup: 1000
+  #    runAsNonRoot: true
+  #    allowPrivilegeEscalation: false
+  #    capabilities:
+  #      drop:
+  #        - ALL
+  #    seccompProfile:
+  #      type: RuntimeDefault
+
 ###
 # Configures `Ingress <https://kubernetes.io/docs/concepts/services-networking/ingress/>`__ for the Tenant S3 API and Console.
 #
@@ -428,7 +451,7 @@ ingress:
     labels: { }
     annotations:
       nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
-      nginx.ingress.kubernetes.io/proxy-body-size: 100m
+      nginx.ingress.kubernetes.io/proxy-body-size: 10000m
     tls: [ ]
     host: minio.local
     path: /
@@ -439,6 +462,7 @@ ingress:
     labels: { }
     annotations:
       nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
+      nginx.ingress.kubernetes.io/proxy-body-size: 10000m
     tls: [ ]
     host: minio-console.local
     path: /
@@ -451,7 +475,7 @@ ingress:
 #    kind: Secret
 #    type: Opaque
 #    metadata:
-#      name: {{ dig "secrets" "existingSecret" "" (.Values | merge (dict)) }}
+#      name: {{ dig "tenant" "configSecret" "name" "" (.Values | merge (dict)) }}
 #    stringData:
 #      config.env: |-
 #        export MINIO_ROOT_USER='minio'

modules/minio/minio-operator.tf

@@ -5,5 +5,5 @@ resource "helm_release" "minio_operator" {
   create_namespace  = "true"
   namespace         = "minio-operator"
   dependency_update = "true"
-  version           = "5.0.12"
+  version           = "6.0.4"
 }

modules/minio/minio-tenant.tf

@@ -6,7 +6,7 @@ resource "helm_release" "minio_tenant" {
   create_namespace  = "true"
   namespace         = "${var.namespace_prefix}minio-tenant"
   dependency_update = "true"
-  version           = "5.0.12"
+  version           = "6.0.4"
 
   values = [
     file("./envs/${var.env}/minio-tenant.yaml")
@@ -21,40 +21,4 @@ resource "helm_release" "minio_tenant" {
     name = "ingress.console.host"
     value = "console-minio.${var.domain}"
   }
-}
+}
-
-/*
-resource "kubernetes_manifest" "minio_ingress" {
-  manifest = yamldecode(<<YAML
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: ingress-minio
-  namespace: block-storage
-  annotations:
-    kubernetes.io/ingress.class: "nginx"
-    ## Remove if using CA signed certificate
-    nginx.ingress.kubernetes.io/proxy-ssl-verify: "off"
-    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
-    nginx.ingress.kubernetes.io/rewrite-target: /
-    nginx.ingress.kubernetes.io/proxy-body-size: "0"
-spec:
-  ingressClassName: nginx
-  tls:
-  - hosts:
-      - minio.${var.domain}
-    secretName: nginx-tls
-  rules:
-  - host: minio.${var.domain}
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: minio
-            port:
-              number: 443
-  YAML
-  )
-}*/