From ad50c8c581adc8bda12be3a1121f0bbc16ca5756 Mon Sep 17 00:00:00 2001 From: Hamza Remmal <hamza@remmal.net> Date: Thu, 23 Jan 2025 12:44:22 +0000 Subject: [PATCH] Some updated evrywhere in the repo :-) --- Makefile | 51 +-- docker-compose.yaml | 2 + docs/dev_setup.md | 129 ++------ k8s/clusters/README.md | 1 - k8s/clusters/lilith@EPFL/README.md | 1 - .../kubernetes-dashboard/README.md | 11 - .../kubernetes-dashboard/dashboard.yaml | 306 ------------------ .../kubernetes-dashboard/ingress.yaml | 29 -- .../kubernetes-dashboard/kustomization.yaml | 8 - k8s/clusters/lilith@EPFL/kustomization.yaml | 6 - k8s/clusters/lilith@EPFL/logging/.gitignore | 3 - k8s/clusters/lilith@EPFL/logging/README.md | 69 ---- k8s/clusters/lilith@EPFL/logging/ingress.yaml | 16 - .../lilith@EPFL/logging/kustomization.yaml | 38 --- .../lilith@EPFL/logging/secrets/.gitkeep | 0 k8s/deployments/EPFL/README.md | 7 - k8s/deployments/EPFL/prod/kustomization.yaml | 7 - k8s/deployments/local/.gitignore | 2 - k8s/deployments/local/README.md | 0 .../autograde-service-configuration.env | 8 - .../local/config/moodle-configuration.env | 7 - k8s/deployments/local/kustomization.yaml | 96 ------ .../secrets/autograde-service-secrets.env | 3 - .../local/secrets/moodle-db-secrets.env | 4 - k8s/xaas-cluster-resources/Chart.yaml | 4 - k8s/xaas-cluster-resources/README.md | 23 -- .../templates/admin-rolebinding.yaml | 18 -- .../templates/ci-rolebinding.yaml | 28 -- .../templates/crds-clusterrole.yaml | 23 -- .../templates/fluentbit-clusterrole.yaml | 29 -- .../templates/gitlabagent-clusterrole.yaml | 25 -- .../templates/namespaces.yaml | 9 - .../readonly-clusterrolebinding.yaml | 31 -- k8s/xaas-cluster-resources/values.yaml | 32 -- 34 files changed, 24 insertions(+), 1002 deletions(-) delete mode 100644 k8s/clusters/README.md delete mode 100644 k8s/clusters/lilith@EPFL/README.md delete mode 100644 k8s/clusters/lilith@EPFL/kubernetes-dashboard/README.md delete mode 100644 k8s/clusters/lilith@EPFL/kubernetes-dashboard/dashboard.yaml delete mode 100644 k8s/clusters/lilith@EPFL/kubernetes-dashboard/ingress.yaml delete mode 100644 k8s/clusters/lilith@EPFL/kubernetes-dashboard/kustomization.yaml delete mode 100644 k8s/clusters/lilith@EPFL/kustomization.yaml delete mode 100644 k8s/clusters/lilith@EPFL/logging/.gitignore delete mode 100644 k8s/clusters/lilith@EPFL/logging/README.md delete mode 100644 k8s/clusters/lilith@EPFL/logging/ingress.yaml delete mode 100644 k8s/clusters/lilith@EPFL/logging/kustomization.yaml delete mode 100644 k8s/clusters/lilith@EPFL/logging/secrets/.gitkeep delete mode 100644 k8s/deployments/local/.gitignore delete mode 100644 k8s/deployments/local/README.md delete mode 100644 k8s/deployments/local/config/autograde-service-configuration.env delete mode 100644 k8s/deployments/local/config/moodle-configuration.env delete mode 100644 k8s/deployments/local/kustomization.yaml delete mode 100644 k8s/deployments/local/secrets/autograde-service-secrets.env delete mode 100644 k8s/deployments/local/secrets/moodle-db-secrets.env delete mode 100644 k8s/xaas-cluster-resources/Chart.yaml delete mode 100644 k8s/xaas-cluster-resources/README.md delete mode 100644 k8s/xaas-cluster-resources/templates/admin-rolebinding.yaml delete mode 100644 k8s/xaas-cluster-resources/templates/ci-rolebinding.yaml delete mode 100644 k8s/xaas-cluster-resources/templates/crds-clusterrole.yaml delete mode 100644 k8s/xaas-cluster-resources/templates/fluentbit-clusterrole.yaml delete mode 100644 k8s/xaas-cluster-resources/templates/gitlabagent-clusterrole.yaml delete mode 100644 k8s/xaas-cluster-resources/templates/namespaces.yaml delete mode 100644 k8s/xaas-cluster-resources/templates/readonly-clusterrolebinding.yaml delete mode 100644 k8s/xaas-cluster-resources/values.yaml diff --git a/Makefile b/Makefile index cad672e6..5d8076eb 100644 --- a/Makefile +++ b/Makefile @@ -1,49 +1,5 @@ SHELL := /bin/bash -.PHONY: start-minikube set-context dashboard build-autograde build-moodle deploy restart-autograde restart-moodle stop-minikube destroy-minikube add-hosts start-tunnel - -start-minikube: - minikube start --driver=docker --cpus=4 --memory=6g --kubernetes-version=v1.28.12 - minikube addons enable metrics-server - -set-context: - kubectl config set-context minikube - -dashboard: - minikube dashboard - -build-autograde: - @eval $$(minikube -p minikube docker-env) && \ - docker build -t autograde/autograde-service:latest-dev autograde-service/ - -build-moodle: - @eval $$(minikube -p minikube docker-env) && \ - docker build -t autograde/moodle:latest-dev -f Dockerfile-moodle-with-autograde ./ - -build-submission-manager: - @eval $$(minikube -p minikube docker-env) && \ - docker build -t autograde/submission-manager:latest-dev autograde-submission-manager/ - -deploy: - kubectl apply -k k8s/deployments/local - -restart-autograde: - kubectl rollout restart deployment autograde-service - -restart-moodle: - kubectl rollout restart statefulset moodle - -update-grading-service-env: - - kubectl delete secrets grading-service-variables - kubectl apply -k k8s/deploy-envs/local - kubectl rollout restart deployment grading-service - -stop-minikube: - minikube stop - -destroy-minikube: - - kubectl delete -k k8s/deployments/local - - minikube stop - - minikube delete +.PHONY: add-host release-autograde release-moodle release-submission-manager add-host: echo "127.0.0.1 moodle" | sudo tee -a /etc/hosts @@ -55,14 +11,11 @@ else sudo sed -i '/moodle/d' /etc/hosts endif -start-tunnel: - sudo minikube tunnel - release-autograde: docker buildx build --platform linux/amd64,linux/arm64 --push -t autograde/autograde-service:1.2.2 autograde-service/ release-moodle: - docker buildx build --platform linux/amd64,linux/arm64 --push -t autograde/moodle:1.2.2 -f Dockerfile-moodle-with-autograde ./ + docker buildx build --platform linux/amd64,linux/arm64 --push -t autograde/moodle:1.2.2 autograde-plugins/assignsubmission_autograde/ release-submission-manager: docker buildx build --platform linux/amd64,linux/arm64 --push -t autograde/submission-manager:1.2.2 autograde-submission-manager/ diff --git a/docker-compose.yaml b/docker-compose.yaml index dee033d1..d0d2056b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -37,6 +37,7 @@ services: networks: - autograde-network kubernetes: + container_name: kubernetes image: rancher/k3s:v1.26.3-k3s1 command: server --disable=traefik,coredns privileged: true @@ -63,6 +64,7 @@ services: ports: - "8082:8082" environment: + AUTOGRADE_API_KEY: 12345 AUTOGRADE_BASEURL: http://192.168.1.100:8082 # NOTE: We hardcode the ip because DNS resolution fails inside pods (but not the node) AUTOGRADE_MOODLE_BASEURL: http://moodle:80 AUTOGRADE_MOODLE_TOKEN: d2d81554afb08a419d38146c8b23f923 diff --git a/docs/dev_setup.md b/docs/dev_setup.md index 60c02950..ff494bbc 100644 --- a/docs/dev_setup.md +++ b/docs/dev_setup.md @@ -6,125 +6,59 @@ The following instructions guide you through the installation and configuration ### Step 0: Required tools -The following tools are required for local experimentations: +The following tools are required for local experimentation: - [docker](https://docs.docker.com/engine/install/), the Docker containers runtime -- [minikube](https://minikube.sigs.k8s.io/docs/start/), to create a local Kubernetes cluster -- [kubectl](https://kubernetes.io/docs/tasks/tools/), the CLI for interacting with Kubernetes cluster - [make](https://www.gnu.org/software/make/manual/make.html), the build automation tool - [git](https://git-scm.com/), the distributed version control system You will first have to clone the project repository with its dependencies ``` -git clone --recursive git@gitlab.epfl.ch:cs107/moodle-autograde.git +git clone git@gitlab.epfl.ch:cs107/moodle-autograde.git ``` -### Step 1: Start minikube +### Step 1: Start The Services -Before starting the minikube cluster, you will have to make sure that the [Docker Engine is running](https://docs.docker.com/config/daemon/start/). If not, please start the *Docker deamon* before running the following command: +Before starting the services, you will have to make sure that the [Docker Engine is running](https://docs.docker.com/config/daemon/start/). +If not, please start the *Docker daemon* before running the following command: ``` -make start-minikube +docker compose up --build ``` -### Step 2: Build the images +> Note that it can take a few minutes after running the command for the services to be fully deployed. -Before deploying the service to your local cluster, you will have to build the Docker images from your local repository. Bare in mind that any changes in the code will require building new images. -``` -make build-moodle -``` - -Build the autograde image: -``` -make build-autograde -``` - -Build the autograde submission manager: -``` -make build-submission-manager -``` - -### Step 3: Deployment - -Now that you have built the project, the next step will be to deploy it to your local minikube cluster. You can run the following Makefile target `deploy` to successfully deploy the services: - -``` -make deploy -``` +### Step 2: Configuring Your Local Domain Names -Note that it can take a few minutes after running the command for the services to be fully deployed. - -### Step 4: Configuring your local domain names - -Link the local instance of Moodle with the host `moodle`. This is done by adding the line `127.0.0.1 moodle` to [/etc/host](https://en.wikipedia.org/wiki/Hosts_(file)): +Link the local instance of Moodle with the host `moodle`. +This is done by adding the line `127.0.0.1 moodle` to [/etc/host](https://en.wikipedia.org/wiki/Hosts_(file)): ``` make add-host ``` -### Step 5: Start the tunnel - -As minikube resides in a Docker container, the cluster's network is isolated from the host network. To access the cluster's services (moodle, autograde, ...), you will need to start a network tunnel between the minikube cluster and your computer. You can create the tunnel by running the following command in a new terminal: - -``` -make start-tunnel -``` - -This commands runs until exited. - -### Step 6: Access the services +### Step 3: Access The Services The local instance of Moodle is accessible at http://moodle. The autograde service is reachable at http://localhost:8082/api/v1/ping. -### Step 7: Moodle installation +### Step 4: Moodle Installation -At this point you should see the following webpage when accessing http://moodle: +At this point, you should see the following webpage when accessing http://moodle:  -Go through the [installation step](https://docs.moodle.org/404/en/Installing_Moodle#Web_based_installer). You can enter whatever value you want for the required fields. +Go through the [installation step](https://docs.moodle.org/404/en/Installing_Moodle#Web_based_installer). +You can enter whatever value you want for the required fields. -### Step 8: Moodle configuration +### Step 5: Moodle Configuration Now that Moodle is installed, follow the instruction in [moodle_config.md](./moodle_config.md) to configure the Moodle instance. -### Step 9: Env configuration - -In the file [local/secrets/autograde-service-secrets.env](../k8s/deployments/local/secrets/autograde-service-secrets.env) set the `AUTOGRADE_MOODLE_TOKEN` to the Moodle token generated in the previous step. The token can be found in Moodle at `Site Administration > Server > Manage Tokens`. - -Reload the autograde service: -``` -make deploy restart-autograde -``` - -At this point you should have a local instance of the Moodle autograde cluster configured and running. +At this point, you should have a local instance of the Moodle autograde cluster configured and running. -### Step 10: To go further +### Step 6: To Go Further -Now that a local instance of the autograde cluster is setup you can experiment with creating classes and autograding homeworks. - -## Stop / Destroy the cluster - -Stop the cluster: -``` -make stop-minikube -``` - -With the stop command, the state of the cluster is saved, and the cluster can be resumed with: -``` -make start-minikube -``` - -Destructive stop, the cluster is stopped and removed: -``` -make destroy-minikube -``` - -## Cluster dashboard - -Get insight about the internals of the cluster: -``` -make dashboard -``` +Now that a local instance of the autograde cluster is set up, +you can experiment with creating classes and auto-graded homeworks. ## Reset the local host configuration @@ -132,26 +66,3 @@ You can remove the mapping from the `moodle` host name with: ``` make rm-host ``` - -## Kubectl context - -Kubectl is the command line tool to interact with a Kubernetes cluster. The command that starts the minikube also configures kubectl to interact with the local cluster. If you work with another Kubernetes cluster and want to switch back the kubectl context to the local autograde cluster, use the following command: -``` -make set-context -``` - -## Development - -During development, if modifying the plugin code or the autograde service you must rebuild the corresponding image and redeploy the cluster to apply the changes. - -After a change to the autograde service: -``` -make build-autograde -make deploy restart-autograde -``` - -After a change to the Moodle plugin: -``` -make build-moodle -make deploy restart-moodle -``` diff --git a/k8s/clusters/README.md b/k8s/clusters/README.md deleted file mode 100644 index 470e6774..00000000 --- a/k8s/clusters/README.md +++ /dev/null @@ -1 +0,0 @@ -Configuration of the clusters used by AUTOGRADE for EPFL \ No newline at end of file diff --git a/k8s/clusters/lilith@EPFL/README.md b/k8s/clusters/lilith@EPFL/README.md deleted file mode 100644 index df656349..00000000 --- a/k8s/clusters/lilith@EPFL/README.md +++ /dev/null @@ -1 +0,0 @@ -Configuration of the kubernetes cluster provided by IC-IT, named lilith \ No newline at end of file diff --git a/k8s/clusters/lilith@EPFL/kubernetes-dashboard/README.md b/k8s/clusters/lilith@EPFL/kubernetes-dashboard/README.md deleted file mode 100644 index f1cd34fe..00000000 --- a/k8s/clusters/lilith@EPFL/kubernetes-dashboard/README.md +++ /dev/null @@ -1,11 +0,0 @@ -### Deploying the Kubernetes web-ui - ---- - -1. Run the following command to deploy an Ingress resource for the dashboard: - - ```bash - kubectl apply -k k8s/deploy-envs/cluster/kubernetes-dashboard - ``` - -2. Access the dashboard at `moodle-autograde.epfl.ch/k8s/`. diff --git a/k8s/clusters/lilith@EPFL/kubernetes-dashboard/dashboard.yaml b/k8s/clusters/lilith@EPFL/kubernetes-dashboard/dashboard.yaml deleted file mode 100644 index a5d341e1..00000000 --- a/k8s/clusters/lilith@EPFL/kubernetes-dashboard/dashboard.yaml +++ /dev/null @@ -1,306 +0,0 @@ -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Namespace -metadata: - name: kubernetes-dashboard - ---- - -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - k8s-app: kubernetes-dashboard - name: kubernetes-dashboard - namespace: kubernetes-dashboard - ---- - -kind: Service -apiVersion: v1 -metadata: - labels: - k8s-app: kubernetes-dashboard - name: kubernetes-dashboard - namespace: kubernetes-dashboard -spec: - ports: - - port: 443 - targetPort: 8443 - selector: - k8s-app: kubernetes-dashboard - ---- - -apiVersion: v1 -kind: Secret -metadata: - labels: - k8s-app: kubernetes-dashboard - name: kubernetes-dashboard-certs - namespace: kubernetes-dashboard -type: Opaque - ---- - -apiVersion: v1 -kind: Secret -metadata: - labels: - k8s-app: kubernetes-dashboard - name: kubernetes-dashboard-csrf - namespace: kubernetes-dashboard -type: Opaque -data: - csrf: "" - ---- - -apiVersion: v1 -kind: Secret -metadata: - labels: - k8s-app: kubernetes-dashboard - name: kubernetes-dashboard-key-holder - namespace: kubernetes-dashboard -type: Opaque - ---- - -kind: ConfigMap -apiVersion: v1 -metadata: - labels: - k8s-app: kubernetes-dashboard - name: kubernetes-dashboard-settings - namespace: kubernetes-dashboard - ---- - -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - labels: - k8s-app: kubernetes-dashboard - name: kubernetes-dashboard - namespace: kubernetes-dashboard -rules: - # Allow Dashboard to get, update and delete Dashboard exclusive secrets. - - apiGroups: [""] - resources: ["secrets"] - resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"] - verbs: ["get", "update", "delete"] - # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map. - - apiGroups: [""] - resources: ["configmaps"] - resourceNames: ["kubernetes-dashboard-settings"] - verbs: ["get", "update"] - # Allow Dashboard to get metrics. - - apiGroups: [""] - resources: ["services"] - resourceNames: ["heapster", "dashboard-metrics-scraper"] - verbs: ["proxy"] - - apiGroups: [""] - resources: ["services/proxy"] - resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"] - verbs: ["get"] - ---- - -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - labels: - k8s-app: kubernetes-dashboard - name: kubernetes-dashboard -rules: - # Allow Metrics Scraper to get metrics from the Metrics server - - apiGroups: ["metrics.k8s.io"] - resources: ["pods", "nodes"] - verbs: ["get", "list", "watch"] - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - k8s-app: kubernetes-dashboard - name: kubernetes-dashboard - namespace: kubernetes-dashboard -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: kubernetes-dashboard -subjects: - - kind: ServiceAccount - name: kubernetes-dashboard - namespace: kubernetes-dashboard - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kubernetes-dashboard -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kubernetes-dashboard -subjects: - - kind: ServiceAccount - name: kubernetes-dashboard - namespace: kubernetes-dashboard - ---- - -kind: Deployment -apiVersion: apps/v1 -metadata: - labels: - k8s-app: kubernetes-dashboard - name: kubernetes-dashboard - namespace: kubernetes-dashboard -spec: - replicas: 1 - revisionHistoryLimit: 10 - selector: - matchLabels: - k8s-app: kubernetes-dashboard - template: - metadata: - labels: - k8s-app: kubernetes-dashboard - spec: - securityContext: - seccompProfile: - type: RuntimeDefault - containers: - - name: kubernetes-dashboard - image: kubernetesui/dashboard:v2.7.0 - imagePullPolicy: Always - ports: - - containerPort: 8443 - protocol: TCP - args: - - --auto-generate-certificates - - --namespace=kubernetes-dashboard - # Uncomment the following line to manually specify Kubernetes API server Host - # If not specified, Dashboard will attempt to auto discover the API server and connect - # to it. Uncomment only if the default does not work. - # - --apiserver-host=http://my-address:port - volumeMounts: - - name: kubernetes-dashboard-certs - mountPath: /certs - # Create on-disk volume to store exec logs - - mountPath: /tmp - name: tmp-volume - livenessProbe: - httpGet: - scheme: HTTPS - path: / - port: 8443 - initialDelaySeconds: 30 - timeoutSeconds: 30 - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - runAsUser: 1001 - runAsGroup: 2001 - volumes: - - name: kubernetes-dashboard-certs - secret: - secretName: kubernetes-dashboard-certs - - name: tmp-volume - emptyDir: {} - serviceAccountName: kubernetes-dashboard - nodeSelector: - "kubernetes.io/os": linux - # Comment the following tolerations if Dashboard must not be deployed on master - tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule - ---- - -kind: Service -apiVersion: v1 -metadata: - labels: - k8s-app: dashboard-metrics-scraper - name: dashboard-metrics-scraper - namespace: kubernetes-dashboard -spec: - ports: - - port: 8000 - targetPort: 8000 - selector: - k8s-app: dashboard-metrics-scraper - ---- - -kind: Deployment -apiVersion: apps/v1 -metadata: - labels: - k8s-app: dashboard-metrics-scraper - name: dashboard-metrics-scraper - namespace: kubernetes-dashboard -spec: - replicas: 1 - revisionHistoryLimit: 10 - selector: - matchLabels: - k8s-app: dashboard-metrics-scraper - template: - metadata: - labels: - k8s-app: dashboard-metrics-scraper - spec: - securityContext: - seccompProfile: - type: RuntimeDefault - containers: - - name: dashboard-metrics-scraper - image: kubernetesui/metrics-scraper:v1.0.8 - ports: - - containerPort: 8000 - protocol: TCP - livenessProbe: - httpGet: - scheme: HTTP - path: / - port: 8000 - initialDelaySeconds: 30 - timeoutSeconds: 30 - volumeMounts: - - mountPath: /tmp - name: tmp-volume - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - runAsUser: 1001 - runAsGroup: 2001 - serviceAccountName: kubernetes-dashboard - nodeSelector: - "kubernetes.io/os": linux - # Comment the following tolerations if Dashboard must not be deployed on master - tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule - volumes: - - name: tmp-volume - emptyDir: {} \ No newline at end of file diff --git a/k8s/clusters/lilith@EPFL/kubernetes-dashboard/ingress.yaml b/k8s/clusters/lilith@EPFL/kubernetes-dashboard/ingress.yaml deleted file mode 100644 index d5bbb8ff..00000000 --- a/k8s/clusters/lilith@EPFL/kubernetes-dashboard/ingress.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: kubernetes-dashboard-ingress - labels: - app: kubernetes-dashboard - annotations: - kubernetes.io/ingress.class: nginx - # backing service is HTTPS - nginx.ingress.kubernetes.io/backend-protocol: HTTPS - # Workaround for https://github.com/kubernetes/dashboard/issues/5017 - nginx.ingress.kubernetes.io/configuration-snippet: | - proxy_set_header Accept-Encoding ""; - sub_filter '<base href="/">' '<base href="/k8s/">'; - sub_filter_once on; - # Enable path rewriting - nginx.ingress.kubernetes.io/rewrite-target: /$2 -spec: - rules: - - host: moodle-autograde.epfl.ch - http: - paths: - - path: /k8s(/|$)(.*) - pathType: Prefix - backend: - service: - name: kubernetes-dashboard - port: - number: 443 diff --git a/k8s/clusters/lilith@EPFL/kubernetes-dashboard/kustomization.yaml b/k8s/clusters/lilith@EPFL/kubernetes-dashboard/kustomization.yaml deleted file mode 100644 index 7e3978ec..00000000 --- a/k8s/clusters/lilith@EPFL/kubernetes-dashboard/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -namespace: kubernetes-dashboard - -resources: - - dashboard.yaml - - ingress.yaml diff --git a/k8s/clusters/lilith@EPFL/kustomization.yaml b/k8s/clusters/lilith@EPFL/kustomization.yaml deleted file mode 100644 index a1a376e2..00000000 --- a/k8s/clusters/lilith@EPFL/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - logging - - kubernetes-dashboard diff --git a/k8s/clusters/lilith@EPFL/logging/.gitignore b/k8s/clusters/lilith@EPFL/logging/.gitignore deleted file mode 100644 index 54f2ebe1..00000000 --- a/k8s/clusters/lilith@EPFL/logging/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ - -# Secret configuration for logging -secrets/openobserve-secrets.env diff --git a/k8s/clusters/lilith@EPFL/logging/README.md b/k8s/clusters/lilith@EPFL/logging/README.md deleted file mode 100644 index 1ef06d00..00000000 --- a/k8s/clusters/lilith@EPFL/logging/README.md +++ /dev/null @@ -1,69 +0,0 @@ -### Deploying log forwarding and log monitoring - ---- - -1. Go to the [`secrets`](secrets/README.md) subdirectory and deploy the secrets first. - -2. Apply the following command to deploy the log forwarding and log monitoring stack: - - ```bash - kubectl apply -k k8s/deploy-envs/cluster/logging - ``` - -3. Access the OpenObserve interface at `moodle-autograde.epfl.ch/logs/web`, - and log-in with the credentials you set in the secrets. - ---- - -4. Update the `fluent-bit` configuration to set the OUTPUT stream to OpenObserve: - - 1. Go to `moodle-autograde.epfl.ch/logs/web/ingestion/logs/fluentbit` and copy the configuration. It will be of the form: - - ```conf - [OUTPUT] - Name http - Match kube.* - URI /api/default/default/_json - Host localhost - Port 5080 - tls Off - Format json - Json_date_key _timestamp - Json_date_format iso8601 - HTTP_User <root user email> - HTTP_Passwd <api key> - compress gzip - ``` - - 2. To split the output stream per namespace: - - 1. Update the `Match` directive to - ```conf - Match kube.var.log.containers.*_<namespace>_* - ``` - 2. Update the `URI` directive to _(remember to add the base prefix `/logs` to the URI)_ - ```conf - URI /logs/api/default/<namespace>/_json - ``` - 3. Update the `Host` directive to - ```conf - Host openobserve.logging - ``` - 4. _Duplicate the `OUTPUT` directive for each namespace you want to monitor._ - - 3. Update the configmap `fluent-bit` in the namespace `logging` with the new configuration. - - 1. Use this command to edit the configmap: - ```bash - kubectl edit configmap fluent-bit -n logging - ``` - 2. In `fluent-bit.conf` value, replace the `[OUTPUT]` directive with the new configuration. - 3. Save the configmap. - 4. Restart the `fluent-bit` DaemonSet with the command: - ```bash - kubectl rollout restart daemonset fluent-bit -n logging - ``` - ---- - -5. You can access the OpenObserve user documentation here, https://openobserve.ai/docs/user-guide/logs/log-search/. diff --git a/k8s/clusters/lilith@EPFL/logging/ingress.yaml b/k8s/clusters/lilith@EPFL/logging/ingress.yaml deleted file mode 100644 index e7fd98b1..00000000 --- a/k8s/clusters/lilith@EPFL/logging/ingress.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: openobserve-ingress -spec: - rules: - - host: moodle-autograde.epfl.ch - http: - paths: - - path: /logs - pathType: Prefix - backend: - service: - name: openobserve - port: - number: 5080 diff --git a/k8s/clusters/lilith@EPFL/logging/kustomization.yaml b/k8s/clusters/lilith@EPFL/logging/kustomization.yaml deleted file mode 100644 index 2fc74f25..00000000 --- a/k8s/clusters/lilith@EPFL/logging/kustomization.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -namespace: logging - -resources: - - ../../../components/logging - - ingress.yaml - -configMapGenerator: - - name: openobserve-configuration - literals: - - ZO_BASE_URI=/logs # should be the same as the prefix in ingress - -secretGenerator: - # Configuration of the openobserve-service - - name: openobserve-secrets - type: Opaque - envs: - - openobserve-secrets.env - -generatorOptions: - disableNameSuffixHash: true - -patches: - # Patch openobserve StatefulSet's volumeClaimTemplates to use class nfs-client - - target: - group: apps - version: v1 - kind: StatefulSet - name: openobserve - patch: |- - - op: replace - path: /spec/volumeClaimTemplates/0/spec/storageClassName - value: nfs-client - - op: replace - path: /spec/volumeClaimTemplates/0/spec/resources/requests/storage - value: 100Gi diff --git a/k8s/clusters/lilith@EPFL/logging/secrets/.gitkeep b/k8s/clusters/lilith@EPFL/logging/secrets/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/k8s/deployments/EPFL/README.md b/k8s/deployments/EPFL/README.md index 0c7efcc5..d4cf2d77 100644 --- a/k8s/deployments/EPFL/README.md +++ b/k8s/deployments/EPFL/README.md @@ -12,10 +12,3 @@ This folder contains the configurations of autograde for EPFL. - [prod](prod) contains the configuration of the production environment - [staging](staging) contains the configuration of the staging environment - -### Cluster - -EPFL provides us with the kubernetes cluster called [`lilith`](../../clusters/lilith@EPFL). -This cluster is managed by the [IC-IT@EPFL](https://www.epfl.ch/schools/ic/it/) team. - -TODO: Add information about the cluster (Number of nodes, CPU, memory, ...) \ No newline at end of file diff --git a/k8s/deployments/EPFL/prod/kustomization.yaml b/k8s/deployments/EPFL/prod/kustomization.yaml index 11b95706..db0149ca 100644 --- a/k8s/deployments/EPFL/prod/kustomization.yaml +++ b/k8s/deployments/EPFL/prod/kustomization.yaml @@ -18,17 +18,10 @@ configMapGenerator: - config/autograde-service-configuration.env secretGenerator: - # Secret configuration of the autograde-service - name: autograde-secrets type: Opaque envs: - secrets/autograde-secrets.env - # Access to ic-registry.epfl.ch/autograde - - name: regcred-deployments - type: kubernetes.io/dockerconfigjson - files: - - secrets/.dockerconfigjson - generatorOptions: disableNameSuffixHash: true diff --git a/k8s/deployments/local/.gitignore b/k8s/deployments/local/.gitignore deleted file mode 100644 index efd12c1f..00000000 --- a/k8s/deployments/local/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ - -kubeconfig diff --git a/k8s/deployments/local/README.md b/k8s/deployments/local/README.md deleted file mode 100644 index e69de29b..00000000 diff --git a/k8s/deployments/local/config/autograde-service-configuration.env b/k8s/deployments/local/config/autograde-service-configuration.env deleted file mode 100644 index 1e88784c..00000000 --- a/k8s/deployments/local/config/autograde-service-configuration.env +++ /dev/null @@ -1,8 +0,0 @@ -AUTOGRADE_BASEURL=http://autograde-service:8082 -AUTOGRADE_MOODLE_BASEURL=http://moodle:80 -AUTOGRADE_JOB_NAMESPACE=default -AUTOGRADE_JOB_IMAGE_PULL-POLICY=IFNOTPRESENT -AUTOGRADE_JOB_TTL=172800 -AUTOGRADE_MANAGER_IMAGE=autograde/submission-manager:latest-dev -AUTOGRADE_MANAGER_PULL-POLICY=NEVER -GRADING_SERVICE_LOG_LEVEL=INFO diff --git a/k8s/deployments/local/config/moodle-configuration.env b/k8s/deployments/local/config/moodle-configuration.env deleted file mode 100644 index 1613c903..00000000 --- a/k8s/deployments/local/config/moodle-configuration.env +++ /dev/null @@ -1,7 +0,0 @@ -MOODLE_DB_HOST=db -MOODLE_DB_NAME=moodle -MOODLE_DB_PASSWORD=moodle -MOODLE_DB_PORT=3306 -MOODLE_DB_TYPE=mysqli -MOODLE_DB_USER=moodle -MOODLE_URL=http://moodle:80 diff --git a/k8s/deployments/local/kustomization.yaml b/k8s/deployments/local/kustomization.yaml deleted file mode 100644 index 9b962d6b..00000000 --- a/k8s/deployments/local/kustomization.yaml +++ /dev/null @@ -1,96 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - ../../components/autograde-service - - ../../components/moodle - -images: - - name: autograde-service - newName: autograde/autograde-service - newTag: latest-dev - - name: moodle-with-autograde - newName: autograde/moodle - newTag: latest-dev - -configMapGenerator: - - name: autograde-configuration - envs: - - config/autograde-service-configuration.env - -secretGenerator: - - name: autograde-secrets - type: Opaque - envs: - - secrets/autograde-service-secrets.env - - name: moodle-configuration - envs: - - config/moodle-configuration.env - - name: moodle-db-configuration - type: Opaque - envs: - - secrets/moodle-db-secrets.env - -generatorOptions: - disableNameSuffixHash: true - -patches: - - target: - kind: Service - name: moodle - patch: | - - path: /spec/type - op: replace - value: LoadBalancer - - target: - kind: Service - name: db - patch: | - - path: /spec/type - op: replace - value: LoadBalancer - - target: - kind: Service - name: autograde-service - patch: | - - path: /spec/type - op: replace - value: LoadBalancer - - path: /spec/ports/0/port - op: replace - value: 8082 - - target: - kind: Deployment - name: autograde-service - patch: | - - path: /spec/template/spec/containers/0/imagePullPolicy - op: replace - value: IfNotPresent - - target: - kind: Deployment - name: autograde-service - patch: | - - path: /spec/template/spec/containers/0/resources/requests/cpu - op: replace - value: 1 - - target: - kind: Deployment - name: autograde-service - patch: | - - path: /spec/template/spec/containers/0/resources/limits/cpu - op: replace - value: 1 - - target: - kind: Deployment - name: autograde-service - patch: | - - path: /spec/template/spec/containers/0/resources/requests/memory - op: replace - value: 1Gi - - target: - kind: Deployment - name: autograde-service - patch: | - - path: /spec/template/spec/containers/0/resources/limits/memory - op: replace - value: 1Gi diff --git a/k8s/deployments/local/secrets/autograde-service-secrets.env b/k8s/deployments/local/secrets/autograde-service-secrets.env deleted file mode 100644 index e88e2174..00000000 --- a/k8s/deployments/local/secrets/autograde-service-secrets.env +++ /dev/null @@ -1,3 +0,0 @@ -AUTOGRADE_API_KEY=12345 -AUTOGRADE_INTEGRITY_SECRET=2t8gq0uDU(hWad*28YcAPp1!#K[Kqd;ZKv9%3R8(yKdkmA:ZwByBV<VMc+1>wqdB -AUTOGRADE_MOODLE_TOKEN=e0f00bd708d2359f4ab83ffc2254f086 diff --git a/k8s/deployments/local/secrets/moodle-db-secrets.env b/k8s/deployments/local/secrets/moodle-db-secrets.env deleted file mode 100644 index 73b0d63c..00000000 --- a/k8s/deployments/local/secrets/moodle-db-secrets.env +++ /dev/null @@ -1,4 +0,0 @@ -MYSQL_DATABASE=moodle -MYSQL_PASSWORD=moodle -MYSQL_ROOT_PASSWORD=moodle -MYSQL_USER=moodle \ No newline at end of file diff --git a/k8s/xaas-cluster-resources/Chart.yaml b/k8s/xaas-cluster-resources/Chart.yaml deleted file mode 100644 index 94d9f1ba..00000000 --- a/k8s/xaas-cluster-resources/Chart.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v2 -name: moodle-autograde-users -description: A Helm chart for Kubernetes to create namespaces and roles needed for the IC cluster, moodle-autograde. -version: 0.1.0 diff --git a/k8s/xaas-cluster-resources/README.md b/k8s/xaas-cluster-resources/README.md deleted file mode 100644 index 9c6add61..00000000 --- a/k8s/xaas-cluster-resources/README.md +++ /dev/null @@ -1,23 +0,0 @@ -### A Helm chart to create namespaces and roles needed for the cluster - ---- - -Install the chart with the following command: - -``` -helm install requested-namespaces-and-roles k8s/ic-cluster-resources -``` - ---- - -- Creates a list of namespaces, listed in `values.yaml` under `namespaces`. - -- For the users, listed in `values.yaml` under `users`, gives admin access to the corresponding namespaces. - - - For the `dev` user, also provide **read-only** access to cluster level resources (nodes, pv, namespaces, etc.). - - - For the `ci` user, given access to manage all resources except `secrets` in all listed namespaces. - -- Creates ClusterRoles and bindings for services that require access to all namespaces, listed in `values.yaml` under `serviceAccounts`. - -- Deploys the [Kubernetes Dashboard](https://raw.githubusercontent.com/kubernetes/dashboard/v2.7.0/aio/deploy/recommended.yaml) with the default recommended settings. diff --git a/k8s/xaas-cluster-resources/templates/admin-rolebinding.yaml b/k8s/xaas-cluster-resources/templates/admin-rolebinding.yaml deleted file mode 100644 index 46ee628d..00000000 --- a/k8s/xaas-cluster-resources/templates/admin-rolebinding.yaml +++ /dev/null @@ -1,18 +0,0 @@ -{{- range $accessLevel, $names := .Values.namespaces }} -{{- range $namespace := $names }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: admin-role-binding - namespace: {{ $namespace }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: admin -subjects: - - kind: User - name: {{ required (printf "User mapping for access level %s is missing in .Values.users" $accessLevel) (index $.Values.users $accessLevel) }} - apiGroup: rbac.authorization.k8s.io -{{- end }} -{{- end }} diff --git a/k8s/xaas-cluster-resources/templates/ci-rolebinding.yaml b/k8s/xaas-cluster-resources/templates/ci-rolebinding.yaml deleted file mode 100644 index 5ecf6c32..00000000 --- a/k8s/xaas-cluster-resources/templates/ci-rolebinding.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: ci-role -rules: -- apiGroups: ["", "apps", "extensions", "batch", "autoscaling", "networking.k8s.io"] - resources: ["pods", "configmaps", "services", "endpoints", "persistentvolumeclaims", "events", "deployments", "statefulsets", "replicasets", "daemonsets", "jobs", "cronjobs", "ingresses", "horizontalpodautoscalers"] - verbs: ["*"] - -{{- range $accessLevel, $names := .Values.namespaces }} -{{- range $namespace := $names }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: ci-role-binding - namespace: {{ $namespace }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ci-role -subjects: - - kind: User - name: {{ required "Fill user in .Values.users.ci to bind with ci role" $.Values.users.ci }} - apiGroup: rbac.authorization.k8s.io -{{- end }} -{{- end }} diff --git a/k8s/xaas-cluster-resources/templates/crds-clusterrole.yaml b/k8s/xaas-cluster-resources/templates/crds-clusterrole.yaml deleted file mode 100644 index 654c66ad..00000000 --- a/k8s/xaas-cluster-resources/templates/crds-clusterrole.yaml +++ /dev/null @@ -1,23 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: crds-manager -rules: - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["get", "list", "watch", "create", "update"] - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: read-cluster -roleRef: - kind: ClusterRole - name: crds-manager-binding - apiGroup: rbac.authorization.k8s.io -subjects: - - kind: User - name: {{ required "Fill user in .Values.users.dev to bind with crd-manager role" .Values.users.dev }} - apiGroup: rbac.authorization.k8s.io diff --git a/k8s/xaas-cluster-resources/templates/fluentbit-clusterrole.yaml b/k8s/xaas-cluster-resources/templates/fluentbit-clusterrole.yaml deleted file mode 100644 index 1e63d54a..00000000 --- a/k8s/xaas-cluster-resources/templates/fluentbit-clusterrole.yaml +++ /dev/null @@ -1,29 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: fluent-bit-role - labels: - app: fluent-bit -rules: - - apiGroups: [""] - resources: ["namespaces", "pods"] - verbs: ["get", "list", "watch"] - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: fluent-bit-rolebinding - labels: - app: fluent-bit -roleRef: - kind: ClusterRole - name: fluent-bit-role - apiGroup: rbac.authorization.k8s.io -subjects: - - kind: ServiceAccount - {{- with .Values.serviceAccounts.fluentbit }} - name: {{ required "Please specify a serviceAccounts.fluentbit.name" .name }} - namespace: {{ required "Please specify a serviceAccounts.fluentbit.namespace" .namespace }} - {{- end }} diff --git a/k8s/xaas-cluster-resources/templates/gitlabagent-clusterrole.yaml b/k8s/xaas-cluster-resources/templates/gitlabagent-clusterrole.yaml deleted file mode 100644 index 5ad634f1..00000000 --- a/k8s/xaas-cluster-resources/templates/gitlabagent-clusterrole.yaml +++ /dev/null @@ -1,25 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: gitlab-agentk-cm -rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: gitlab-agentk-cm-binding -roleRef: - name: gitlab-agentk-cm - kind: ClusterRole - apiGroup: rbac.authorization.k8s.io -subjects: - - kind: ServiceAccount - {{- with .Values.serviceAccounts.gitlabAgent }} - name: {{ required "Please specify a serviceAccounts.gitlabAgent.name" .name }} - namespace: {{ required "Please specify a serviceAccounts.gitlabAgent.namespace" .namespace }} - {{- end }} diff --git a/k8s/xaas-cluster-resources/templates/namespaces.yaml b/k8s/xaas-cluster-resources/templates/namespaces.yaml deleted file mode 100644 index 699893c3..00000000 --- a/k8s/xaas-cluster-resources/templates/namespaces.yaml +++ /dev/null @@ -1,9 +0,0 @@ -{{- range $accessLevel, $names := .Values.namespaces }} -{{- range $namespace := $names }} ---- -apiVersion: v1 -kind: Namespace -metadata: - name: {{ $namespace }} -{{- end }} -{{- end }} diff --git a/k8s/xaas-cluster-resources/templates/readonly-clusterrolebinding.yaml b/k8s/xaas-cluster-resources/templates/readonly-clusterrolebinding.yaml deleted file mode 100644 index dc2ab007..00000000 --- a/k8s/xaas-cluster-resources/templates/readonly-clusterrolebinding.yaml +++ /dev/null @@ -1,31 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cluster-reader -rules: - - apiGroups: ["*"] - resources: - [ - "namespaces", - "nodes", - "persistentvolumes", - "customresourcedefinitions", - "ingressclasses", - "storageclasses", - ] - verbs: ["get", "list", "watch"] - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cluster-reader-binding -roleRef: - kind: ClusterRole - name: cluster-reader - apiGroup: rbac.authorization.k8s.io -subjects: - - kind: User - name: {{ required "Fill user in .Values.users.dev to bind with cluster-reader role" .Values.users.dev }} - apiGroup: rbac.authorization.k8s.io diff --git a/k8s/xaas-cluster-resources/values.yaml b/k8s/xaas-cluster-resources/values.yaml deleted file mode 100644 index 42e434ef..00000000 --- a/k8s/xaas-cluster-resources/values.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# these users need to be created in the cluster before running helm install -users: - dev: devAdmin - staging: stagingAdmin - prod: prodAdmin - ci: gitlab - -namespaces: - dev: - # for staging master branch - - staging-autograde - # for logging and monitoring services (fluentbit, openobserve, etc.) - - logging - # for gitlab agentk - - gitlab-agent - # for gitlab runner - - gitlab-runner - staging: - # for grading-service intergration with external services - - staging-epfl - prod: - # for production deployment of grading-service - - prod - -# service accounts referenced in clusterrolebindings -serviceAccounts: - fluentbit: - name: fluentbit - namespace: logging - gitlabAgent: - name: gitlab-agentk - namespace: gitlab-agent -- GitLab