#!/bin/sh
set -eu

run_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
# shellcheck disable=SC1090
. "$run_dir/effective_config.env"

: "${HOST_MODEL_ROOT:?set HOST_MODEL_ROOT to the host directory mounted at /opt/local-models}"
: "${HOST_HF_CACHE:?set HOST_HF_CACHE to the host Hugging Face cache directory}"
: "${HOST_PATCH_BUNDLE_PATH:=$patch_bundle_path}"
: "${HOST_RUNTIME_ROOT:=$run_dir/runtime}"
: "${VNEXT_CONTAINER_NAME:=vnext_${profile_name}}"
: "${VNEXT_REPLACE_CONTAINER:=1}"

case "$tp_size" in
    4) default_devices=0,1,2,3 ;;
    8) default_devices=0,1,2,3,4,5,6,7 ;;
    *) default_devices=0 ;;
esac
: "${HIP_VISIBLE_DEVICES:=$default_devices}"

resolve_symlink_target() {
    path=$1
    target=$(readlink "$path") || return 1
    case "$target" in
        /*)
            printf '%s\n' "$target"
            ;;
        *)
            path_dir=$(CDPATH= cd -- "$(dirname -- "$path")" && pwd)
            printf '%s/%s\n' "$path_dir" "$target"
            ;;
    esac
}

has_safetensors_symlink() {
    model_dir=$1
    for shard in "$model_dir"/*.safetensors; do
        [ -L "$shard" ] && return 0
    done
    return 1
}

resolve_hf_snapshot_blobs() {
    model_dir=$1
    case "$model_dir" in
        */snapshots/*) ;;
        *) return 1 ;;
    esac
    blobs_dir=$model_dir/../../blobs
    [ -d "$blobs_dir" ] || return 1
    CDPATH= cd -- "$blobs_dir" && pwd -P
}

mkdir -p \
    "$HOST_HF_CACHE" \
    "$HOST_RUNTIME_ROOT/root/.cache/vllm" \
    "$HOST_RUNTIME_ROOT/root/.triton" \
    "$HOST_RUNTIME_ROOT/tmp/torchinductor_root"

model_root_mount=$HOST_MODEL_ROOT
model_is_symlink=0
model_suffix=
resolved_model_path=
resolved_blobs_path=

case "$model" in
    /opt/local-models/*)
        model_suffix=${model#/opt/local-models/}
        host_model_path=$HOST_MODEL_ROOT/$model_suffix
        if [ -L "$host_model_path" ]; then
            resolved_model_path=$(resolve_symlink_target "$host_model_path") || {
                printf '%s\n' "error: could not resolve symlinked model path: $host_model_path" >&2
                exit 2
            }
            [ -d "$resolved_model_path" ] || {
                printf '%s\n' "error: symlinked model path is not a directory: $host_model_path" >&2
                exit 2
            }

            model_is_symlink=1
            model_root_mount=$HOST_RUNTIME_ROOT/model-bind-root/$profile_name
            model_link=$model_root_mount/$model_suffix
            rm -rf "$model_root_mount"
            mkdir -p "$(dirname -- "$model_link")"
            ln -s "/opt/vnext-models/$model_suffix" "$model_link"

            if has_safetensors_symlink "$resolved_model_path"; then
                resolved_blobs_path=$(resolve_hf_snapshot_blobs "$resolved_model_path") || {
                    printf '%s\n' "error: symlinked HF snapshot shards require a readable blobs directory: $host_model_path" >&2
                    exit 2
                }
            fi
        fi
        ;;
esac

if [ "$VNEXT_REPLACE_CONTAINER" = "1" ]; then
    docker stop "$VNEXT_CONTAINER_NAME" >/dev/null 2>&1 || true
    docker rm "$VNEXT_CONTAINER_NAME" >/dev/null 2>&1 || true
fi

set -- docker run -d \
    --name "$VNEXT_CONTAINER_NAME" \
    --network host \
    --ipc host \
    --privileged \
    --device /dev/kfd \
    --device /dev/dri \
    --group-add video \
    --cap-add SYS_PTRACE \
    --cap-add IPC_LOCK \
    --security-opt seccomp=unconfined \
    --security-opt label=disable \
    --ulimit memlock=-1:-1 \
    -v "$HOST_HF_CACHE:/root/.cache/huggingface" \
    -v "$model_root_mount:/opt/local-models:ro" \
    -v "$run_dir:/runwork" \
    -v "$HOST_RUNTIME_ROOT/root/.cache/vllm:/root/.cache/vllm" \
    -v "$HOST_RUNTIME_ROOT/root/.triton:/root/.triton" \
    -v "$HOST_RUNTIME_ROOT/tmp/torchinductor_root:/tmp/torchinductor_root" \
    -e HIP_VISIBLE_DEVICES="$HIP_VISIBLE_DEVICES" \
    -e LD_LIBRARY_PATH=/opt/gfx906:/rccl-overlay/install/lib:/usr/local/lib:/opt/rocm/lib:/opt/venv/lib/python3.12/site-packages/torch/lib \
    --entrypoint /runwork/container_entrypoint.sh

if [ "$model_is_symlink" = "1" ]; then
    set -- "$@" -v "$resolved_model_path:/opt/vnext-models/$model_suffix:ro"
    if [ -n "$resolved_blobs_path" ]; then
        set -- "$@" -v "$resolved_blobs_path:/opt/blobs:ro"
    fi
fi

if [ -n "$HOST_PATCH_BUNDLE_PATH" ]; then
    set -- "$@" \
        -v "$HOST_PATCH_BUNDLE_PATH:/opt/vllm_patch_bundle:ro" \
        -e VLLM_PATCH_BUNDLE=/opt/vllm_patch_bundle
fi

set -- "$@" "$runtime_image"

"$@" > "$run_dir/container_id.txt"

docker logs -f "$VNEXT_CONTAINER_NAME" > "$run_dir/container.log" 2>&1 &
echo "$!" > "$run_dir/log_tail_pid.txt"
echo "$VNEXT_CONTAINER_NAME" > "$run_dir/container_name.txt"
printf '%s\n' "$VNEXT_CONTAINER_NAME"
