모델 서빙 인프라

FROM busybox:latest

# 모델 가중치 -- safetensors 샤드 및 인덱스
COPY <model-name>/model-*.safetensors        /models/
COPY <model-name>/model.safetensors.index.json /models/

# 모델 구성
COPY <model-name>/config.json            /models/
COPY <model-name>/generation_config.json /models/

# 토크나이저
COPY <model-name>/tokenizer.json          /models/
COPY <model-name>/tokenizer_config.json   /models/
COPY <model-name>/special_tokens_map.json /models/

FROM busybox:latest
COPY <model-name>/ /models/

<registry>/<project>/<model-name>:latest

oc new-project <namespace>

# KServe가 프라이빗 레지스트리에서 모델 이미지를 풀할 수 있도록 하는 풀 시크릿
oc create secret docker-registry model-registry-secret \
  --docker-server=<registry> \
  --docker-username=<username> \
  --docker-password=<password-or-token> \
  -n <namespace>

# KServe predictor에서 사용하는 서비스 계정
oc create sa model-puller-sa -n <namespace>

# 풀 시크릿 연결 -- 두 명령 모두 필요합니다:
# oc secrets link는 일반적인 시크릿 사용을 처리하고;
# imagePullSecrets는 KServe의 ModelCar init 컨테이너에 특별히 필요합니다
oc secrets link model-puller-sa model-registry-secret --for=pull -n <namespace>
oc patch serviceaccount model-puller-sa -n <namespace> \
  -p '{"imagePullSecrets": [{"name": "model-registry-secret"}]}'

apiVersion: serving.kserve.io/v1alpha1
kind: ServingRuntime
metadata:
  name: vllm-runtime
  namespace: <namespace>
spec:
  multiModel: false
  supportedModelFormats:
    - name: pytorch
      autoSelect: true
  containers:
    - name: kserve-container
      image: vllm/vllm-openai:<version>
      ports:
        - containerPort: 3000
          protocol: TCP
      livenessProbe:
        httpGet:
          path: /health
          port: 3000
        periodSeconds: 30
        timeoutSeconds: 5
        failureThreshold: 3
      readinessProbe:
        httpGet:
          path: /health
          port: 3000
        periodSeconds: 10
        timeoutSeconds: 5
        failureThreshold: 3
      startupProbe:
        httpGet:
          path: /health
          port: 3000
        periodSeconds: 10
        timeoutSeconds: 5
        failureThreshold: 60

oc apply -n <namespace> -f serving-runtime-vllm.yaml

apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
  name: <model-name>
  annotations:
    serving.kserve.io/autoscalerClass: external
    serving.kserve.io/deploymentMode: RawDeployment
spec:
  predictor:
    affinity:
      nodeAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
          nodeSelectorTerms:
            - matchExpressions:
                - key: kubernetes.io/arch
                  operator: In
                  values:
                    - amd64
    tolerations:
      - key: nvidia.com/gpu
        operator: Exists
        effect: NoSchedule
    volumes:
      - name: shm
        emptyDir:
          medium: Memory
          sizeLimit: 64Gi
    model:
      modelFormat:
        name: pytorch
      runtime: vllm-runtime
      storageUri: "oci://<registry>/<namespace-or-project>/<model-name>:latest"
      resources:
        requests:
          cpu: "<cpu-request>"          # 예: "8"
        limits:
          cpu: "<cpu-limit>"            # 예: "16"
          memory: <memory-limit>        # 예: 96Gi -- 모델 VRAM 요구 사항에 맞게 설정
          nvidia.com/gpu: "<gpu-count>" # 예: "1"
      volumeMounts:
        - name: shm
          mountPath: /dev/shm
      args:
        - /mnt/models/
        - --served-model-name=<model-name>
        - --port=3000
        - --enable-auto-tool-choice
        - --tool-call-parser=openai
      env:
        - name: HOME
          value: /tmp
        - name: MAX_LOG_LEN
          value: "100"        # vLLM 로그 라인이 로그 범람을 방지하도록 잘라냄
        - name: HF_HUB_CACHE
          value: /tmp
        - name: TRITON_CACHE_DIR
          value: /tmp
        - name: XDG_CACHE_HOME
          value: /tmp
        - name: HF_HOME
          value: /tmp/hf_home
        - name: NUM_GPUS
          value: "<gpu-count>" # 위의 nvidia.com/gpu 제한과 일치해야 합니다
        - name: CUDA_VISIBLE_DEVICES
          value: "<gpu-indices>" # 예: 단일 GPU의 경우 "0"; 두 GPU의 경우 "0,1"
        - name: VLLM_WORKER_MULTIPROC_METHOD
          value: spawn
        - name: LOGNAME
          value: vllm
        - name: USER
          value: vllm

oc apply -n <namespace> -f isvc-<model-name>.yaml

# InferenceService가 Ready 상태에 도달할 때까지 감시
oc get inferenceservice <model-name> -n <namespace> -w

# predictor 파드 시작 감시
oc get pods -n <namespace> -w

# predictor 로그 추적 (Running 상태 이후 모델 로드에 수 분이 소요될 수 있음)
oc logs -f deployment/<model-name>-predictor -n <namespace>

POD=$(oc get pods -n <namespace> \
  -l app=isvc.<model-name>-predictor \
  -o jsonpath='{.items[0].metadata.name}')

oc exec -n <namespace> "$POD" -c kserve-container -- \
  curl -fsS http://127.0.0.1:3000/v1/models

oc exec -n <namespace> "$POD" -c kserve-container -- \
  curl -fsS http://127.0.0.1:3000/v1/chat/completions \
  -H 'Content-Type: application/json' \
  -d '{
    "model": "<model-name>",
    "messages": [{"role": "user", "content": "Hello"}],
    "max_tokens": 50
  }'