# swap 비활성화
sudo swapoff -a

# fstab 수정 (재부팅 시에도 유지)
sudo sed -i '/swap/s/^/#/' /etc/fstab

# NFS 클라이언트 설치
sudo apt-get update && sudo apt-get install -y nfs-common curl

sudo modprobe br_netfilter
sudo modprobe overlay
sudo modprobe nf_conntrack

# 재부팅 후에도 유지
sudo tee /etc/modules-load.d/k8s.conf <<'EOF'
br_netfilter
overlay
nf_conntrack
EOF

sudo tee /etc/sysctl.d/99-k8s-tune.conf <<'EOF'
# K8s 네트워킹 필수
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1

# Connection tracking — GKE/EKS 기본값
net.netfilter.nf_conntrack_max      = 1048576

# Socket backlog
net.core.somaxconn                  = 32768

# TCP TIME_WAIT 소켓 재사용
net.ipv4.tcp_tw_reuse               = 1

# inotify — kubeadm 권장값
fs.inotify.max_user_watches         = 524288
fs.inotify.max_user_instances       = 8192
fs.inotify.max_queued_events        = 32768
EOF

sudo sysctl -p /etc/sysctl.d/99-k8s-tune.conf

sudo mkdir -p /etc/systemd/journald.conf.d
sudo tee /etc/systemd/journald.conf.d/99-k8s-limit.conf <<'EOF'
[Journal]
SystemMaxUse=100M
RuntimeMaxUse=100M
EOF

sudo systemctl restart systemd-journald

journalctl --disk-usage

# kube-vip 등 VIP가 있으면 VIP로, 없으면 master-1 IP로
API_SERVER_IP="<VIP 또는 MASTER01_IP>"

# master-1 에서 실행
sudo curl -sfL https://get.K3s.io | sh -s - server \
  --cluster-init \
  --tls-san=$API_SERVER_IP \
  --disable=traefik \
  --disable=servicelb \
  --flannel-backend=vxlan \
  --etcd-expose-metrics \
  --node-taint node-role.kubernetes.io/control-plane=:NoSchedule \
  --kube-scheduler-arg=bind-address=0.0.0.0 \
  --kube-scheduler-arg=authorization-always-allow-paths=/metrics \
  --kubelet-arg=kube-reserved=cpu=500m,memory=1Gi \
  --kubelet-arg=system-reserved=cpu=200m,memory=256Mi \
  --kubelet-arg=eviction-hard=memory.available<500Mi,nodefs.available<10%,imagefs.available<10%,nodefs.inodesFree<5% \
  --kubelet-arg=container-log-max-size=10Mi \
  --kubelet-arg=container-log-max-files=3 \
  --kubelet-arg=image-gc-high-threshold=80 \
  --kubelet-arg=image-gc-low-threshold=70

# 토큰 조회 (master-2, master-3 조인 시 사용)
sudo cat /var/lib/rancher/K3s/server/node-token
# 예: K10ea6ad1b45...::server:7604d858...

# master-2, master-3 각각에서 실행
sudo curl -sfL https://get.K3s.io | K3s_TOKEN="<MASTER01_TOKEN>" sh -s - server \
  --server https://$API_SERVER_IP:6443 \
  --disable=traefik \
  --disable=servicelb \
  --flannel-backend=vxlan \
  --etcd-expose-metrics \
  --node-taint node-role.kubernetes.io/control-plane=:NoSchedule \
  --kube-scheduler-arg=bind-address=0.0.0.0 \
  --kube-scheduler-arg=authorization-always-allow-paths=/metrics \
  --kubelet-arg=kube-reserved=cpu=500m,memory=1Gi \
  --kubelet-arg=system-reserved=cpu=200m,memory=256Mi \
  --kubelet-arg=eviction-hard=memory.available<500Mi,nodefs.available<10%,imagefs.available<10%,nodefs.inodesFree<5% \
  --kubelet-arg=container-log-max-size=10Mi \
  --kubelet-arg=container-log-max-files=3 \
  --kubelet-arg=image-gc-high-threshold=80 \
  --kubelet-arg=image-gc-low-threshold=70

sudo K3s kubectl get nodes

sudo tee -a /etc/rancher/K3s/config.yaml <<'EOF'

# etcd 스냅샷 보관 설정
etcd-snapshot-retention: 3
etcd-snapshot-schedule-cron: "0 */6 * * *"
EOF

sudo systemctl restart K3s

# 현재 스냅샷 목록
sudo K3s etcd-snapshot ls

# 스냅샷 디렉터리 크기 확인
sudo du -sh /var/lib/rancher/K3s/server/db/snapshots/

sudo curl -sfL https://get.K3s.io | K3s_TOKEN="<MASTER01_TOKEN>" sh -s - agent \
  --server https://$API_SERVER_IP:6443 \
  --kubelet-arg=kube-reserved=cpu=200m,memory=256Mi \
  --kubelet-arg=system-reserved=cpu=200m,memory=256Mi \
  --kubelet-arg=eviction-hard=memory.available<500Mi,nodefs.available<10%,imagefs.available<10%,nodefs.inodesFree<5% \
  --kubelet-arg=container-log-max-size=10Mi \
  --kubelet-arg=container-log-max-files=3 \
  --kubelet-arg=image-gc-high-threshold=80 \
  --kubelet-arg=image-gc-low-threshold=70

mkdir -p ~/.kube
sudo cp /etc/rancher/K3s/K3s.yaml ~/.kube/config
sudo chown $USER:$USER ~/.kube/config

# 환경 변수 및 자동 완성 적용
echo 'export KUBECONFIG=~/.kube/config' >> ~/.bashrc
source <(kubectl completion bash)

kubectl describe nodes master-1 master-2 master-3 | grep Taints

# 클러스터 정보 확인
kubectl cluster-info

# 노드 상태 확인 (Master 3개 HA, Worker 3개)
kubectl get nodes

sudo apt-get update && sudo apt-get install -y nfs-common

helm repo add nfs-ganesha-server-and-external-provisioner https://kubernetes-sigs.github.io/nfs-ganesha-server-and-external-provisioner/
helm repo update

cat > nfs-v4-values.yaml <<'EOF'
# 기존 설정 유지
persistence:
  enabled: true
  storageClass: local-path
  size: 50Gi

storageClass:
  name: nfs
  defaultClass: true
  # NFS v4.1로 변경 (핵심!)
  mountOptions:
    - nfsvers=4.1
    - hard
    - timeo=600
    - retrans=2
    - rsize=1048576
    - wsize=1048576
    - noatime
    - nodiratime

nodeSelector:
  node-role.kubernetes.io/control-plane: "true"

tolerations:
  - key: node-role.kubernetes.io/control-plane
    operator: Exists
    effect: NoSchedule
EOF

helm install nfs-server nfs-ganesha-server-and-external-provisioner/nfs-server-provisioner \
  --namespace kube-system \
  -f nfs-v4-values.yaml

kubectl patch storageclass local-path -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"false"}}}'

helm repo add metallb https://metallb.github.io/metallb
helm repo update
helm install metallb metallb/metallb \
    --version 0.14.9 \
    --namespace metallb-system \
    --create-namespace

# MetalLB 리소스 배포
kubectl apply -f - <<EOF
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
  name: ip-pool
  namespace: metallb-system
spec:
  addresses:
  -  192.168.0.220-192.168.0.230
  autoAssign: true
---
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
  name: l2-network
  namespace: metallb-system
spec:
  ipAddressPools:
    - ip-pool
EOF

# L2Advertisement 대신 아래 리소스를 적용
kubectl apply -f - <<EOF
apiVersion: metallb.io/v1beta2
kind: BGPPeer
metadata:
  name: router
  namespace: metallb-system
spec:
  myASN: 64512          # 클러스터 ASN (사설: 64512-65534)
  peerASN: 64513        # 라우터 ASN
  peerAddress: 192.168.0.1  # 라우터 IP
---
apiVersion: metallb.io/v1beta1
kind: BGPAdvertisement
metadata:
  name: bgp-network
  namespace: metallb-system
spec:
  ipAddressPools:
    - ip-pool
EOF

rm -rf kustomize/base/helm
helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
helm repo update
helm pull ingress-nginx/ingress-nginx \
    --version 4.14.1 \
    --untar \
    --destination kustomize/base/helm

# Nginx Ingress 설치
helm install ingress-nginx ingress-nginx/ingress-nginx \
    --version 4.14.1 \
    --namespace ingress-nginx \
    --create-namespace \
    --values nginx-values.yaml

cat > nginx-values.yaml <<EOF
fullnameOverride: ingress-nginx

controller:
  replicaCount: 1
  progressDeadlineSeconds: 600 
  service:
    type: LoadBalancer
    annotations:
      service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
      service.beta.kubernetes.io/aws-load-balancer-target-type: ip
  resources:
    requests:
      cpu: 100m
      memory: 90Mi
  admissionWebhooks:
    enabled: true
    certManager:
      enabled: false
      clusterIssuer: "selfsigned-issuer"
  config:
    allow-snippet-annotations: "true"
    annotations-risk-level: "Critical"

  ingressClass: nginx
  ingressClassResource:
    name: nginx
    enabled: true
    default: true   # default

  extraArgs:
    watch-ingress-without-class: "true"
EOF

# Nginx 애플리케이션 배포
kubectl create deployment nginx-test --image=nginx
kubectl expose deployment nginx-test --port=80

# 배포 확인
kubectl get pods
kubectl get svc

cat <<EOF | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: nginx-test-ingress
  annotations:
    nginx.ingress.kubernetes.io/rewrite-target: /
spec:
  ingressClassName: nginx
  rules:
  - host: test.cnapcloud.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: nginx-test
            port:
              number: 80
EOF

kubectl get ingress

# Nginx Ingress 서비스의 EXTERNAL-IP 확인
kubectl get svc -n ingress-nginx

# 예: EXTERNAL-IP가 192.168.0.221이라고 가정
# 로컬 호스트 파일에 등록 (관리자 권한 필요)
echo "192.168.0.221 test.cnapcloud.com" | sudo tee -a /etc/hosts

# curl로 테스트
curl http://test.cnapcloud.com

# 또는 웹 브라우저에서 http://test.cnapcloud.com 접속

sudo K3s kubectl get nodes

etcd-expose-metrics: true
kube-scheduler-arg:
  - "bind-address=0.0.0.0"
  - "authorization-always-allow-paths=/metrics"

sudo systemctl restart K3s

curl -L http://<MASTER_IP>:2381/metrics | head -n 10
curl -k https://<MASTER_IP>:10259/metrics | head -n 10

scrape_configs:
  - job_name: 'K3s-etcd'
    static_configs:
      - targets: ['<MASTER_IP>:2381']

# 노드 상태 확인 (Master 3개 HA, Worker 3개)
kubectl get nodes

# 모든 파드 상태 확인
kubectl get pods -A

# StorageClass 확인
kubectl get sc

# Ingress 확인
kubectl get ingress

# 노드 전체 디스크 사용량
df -h

# K3s 데이터 디렉터리별 사용량
sudo du -sh /var/lib/rancher/K3s/server/db/snapshots/   # etcd 스냅샷
sudo du -sh /var/lib/rancher/K3s/agent/containerd/       # 컨테이너 이미지/레이어
sudo du -sh /var/log/pods/                               # 컨테이너 로그 심볼릭 링크

# journald 사용량
journalctl --disk-usage

# 미사용 이미지 전체 삭제
sudo K3s crictl rmi --prune

# 현재 이미지 목록 확인
sudo K3s crictl images

# 현재 실행 중인 컨테이너 확인
sudo K3s crictl ps

sudo tee -a /etc/rancher/K3s/config.yaml <<'EOF'

kubelet-arg:
  - "kube-reserved=cpu=500m,memory=1Gi"    # master: 500m/1Gi | worker: 200m/256Mi
  - "system-reserved=cpu=200m,memory=256Mi"
  - "eviction-hard=memory.available<500Mi,nodefs.available<10%,imagefs.available<10%,nodefs.inodesFree<5%"
  - "container-log-max-size=10Mi"
  - "container-log-max-files=3"
  - "image-gc-high-threshold=80"
  - "image-gc-low-threshold=70"
EOF

# master
sudo systemctl restart K3s
# worker
sudo systemctl restart K3s-agent

sudo cat /proc/$(pgrep -f "K3s server")/cmdline | tr '\0' '\n' | grep kubelet