Skip to content

vllm.entrypoints.pooling.scoring.serving

Classes:

ServingScores

Bases: PoolingServing

Source code in vllm/entrypoints/pooling/scoring/serving.py
class ServingScores(PoolingServing):
    request_id_prefix = "score"

    def __init__(
        self,
        engine_client: EngineClient,
        *args,
        supported_tasks: tuple[SupportedTask, ...],
        enable_flash_late_interaction: bool = True,
        **kwargs,
    ):
        pooling_task = engine_client.model_config.get_pooling_task(supported_tasks)
        score_type = SCORE_TYPE_MAP.get(pooling_task, None)  # type: ignore[arg-type]
        assert score_type is not None

        self.io_processor_name: str = score_type
        self.enable_flash_late_interaction = (
            self.io_processor_name == "late-interaction"
            and enable_flash_late_interaction
        )

        if self.enable_flash_late_interaction:
            self.io_processor_name = "flash-late-interaction"

        if engine_client.model_config.architecture == "JinaForRanking":
            self.io_processor_name = "jina-reranking-scoring"
            self.enable_flash_late_interaction = False

        super().__init__(engine_client, *args, **kwargs)

    def init_io_processor(self, *args, **kwargs) -> PoolingIOProcessor:
        return ScoringIOProcessors[self.io_processor_name](*args, **kwargs)

    async def __call__(self, *args, **kwargs) -> Response:
        if not self.enable_flash_late_interaction:
            return await super().__call__(*args, **kwargs)

        return await self.flash_late_interaction(*args, **kwargs)

    def _build_response(
        self,
        ctx: ScoringServeContext,
    ) -> JSONResponse:
        final_res_batch = ctx.final_res_batch
        request_id = ctx.request_id
        created_time = ctx.created_time
        model_name = ctx.model_name
        assert ctx.engine_inputs is not None
        num_prompt_tokens = sum(
            self._extract_prompt_len(engine_input["prompts"])
            for engine_input in ctx.engine_inputs
        )

        if isinstance(ctx.request, ScoreRequest):
            return self._request_output_to_score_response(
                final_res_batch,
                request_id,
                created_time,
                model_name,
                num_prompt_tokens,
            )
        elif isinstance(ctx.request, RerankRequest):
            return self._request_output_to_rerank_response(
                final_res_batch,
                request_id,
                model_name,
                ctx.request.documents,
                ctx.request.top_n if ctx.request.top_n > 0 else len(final_res_batch),
                num_prompt_tokens,
            )
        else:
            raise ValueError(f"Invalid {self.request_id_prefix} request type")

    def _request_output_to_score_response(
        self,
        final_res_batch: list[PoolingRequestOutput],
        request_id: str,
        created_time: int,
        model_name: str,
        num_prompt_tokens: int,
    ) -> JSONResponse:
        items: list[ScoreResponseData] = []

        for idx, final_res in enumerate(final_res_batch):
            classify_res = ScoringRequestOutput.from_base(final_res)

            item = ScoreResponseData(
                index=idx,
                score=classify_res.outputs.score,
            )
            items.append(item)

        usage = UsageInfo(
            prompt_tokens=num_prompt_tokens,
            total_tokens=num_prompt_tokens,
        )

        response = ScoreResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            data=items,
            usage=usage,
        )

        return JSONResponse(content=response.model_dump())

    def _request_output_to_rerank_response(
        self,
        final_res_batch: list[PoolingRequestOutput],
        request_id: str,
        model_name: str,
        documents: ScoreInput | list[ScoreInput],
        top_n: int,
        num_prompt_tokens: int,
    ) -> JSONResponse:
        if not isinstance(documents, list):
            documents = [documents]

        results: list[RerankResult] = []
        for idx, final_res in enumerate(final_res_batch):
            classify_res = ScoringRequestOutput.from_base(final_res)

            document = documents[idx]
            if isinstance(document, str):
                rerank_document = RerankDocument(text=document)
            else:
                rerank_document = RerankDocument(
                    multi_modal=document.get("content", [])
                )

            result = RerankResult(
                index=idx,
                document=rerank_document,
                relevance_score=classify_res.outputs.score,
            )
            results.append(result)

        # sort by relevance, then return the top n if set
        results.sort(key=lambda x: x.relevance_score, reverse=True)
        if top_n < len(documents):
            results = results[:top_n]

        response = RerankResponse(
            id=request_id,
            model=model_name,
            results=results,
            usage=RerankUsage(
                total_tokens=num_prompt_tokens, prompt_tokens=num_prompt_tokens
            ),
        )

        return JSONResponse(content=response.model_dump())

    ###################################################################################
    ### Run pooling score MaxSim on worker side (GPU) in the API server process
    ### Can significantly improve late-interaction scoring performance.

    async def flash_late_interaction(self, *args, **kwargs) -> Response:
        ctx = await self._init_ctx(self.io_processor, *args, **kwargs)
        await self._preprocessing(self.io_processor, ctx)

        try:
            # stage 1: encode queries and cache token embeddings on workers.
            await self._flash_late_interaction_encode_queries(ctx)
            # stage 2: encode docs and return scalar scores from workers.
            await self._flash_late_interaction_encode_docs(ctx)
        except BaseException:
            cleanup = asyncio.create_task(self._cleanup_flash_late_interaction(ctx))
            try:
                await self._await_late_interaction_cleanup(cleanup)
            except Exception:
                logger.exception("Failed to clean up late-interaction query cache.")
            raise

        return await self._postprocessing_async(self.io_processor, ctx)

    @staticmethod
    async def _await_late_interaction_cleanup(task: asyncio.Future[Any]) -> None:
        """Bound cleanup waiting and defer cancellation until it finishes."""
        loop = asyncio.get_running_loop()
        deadline = loop.time() + _LATE_INTERACTION_CLEANUP_TIMEOUT_SECONDS
        cancellation: asyncio.CancelledError | None = None
        try:
            while not task.done():
                remaining = deadline - loop.time()
                if remaining <= 0:
                    task.cancel()
                    task.add_done_callback(
                        lambda done: None if done.cancelled() else done.exception()
                    )
                    raise TimeoutError("Timed out waiting for late-interaction cleanup")
                try:
                    await asyncio.wait({task}, timeout=remaining)
                except asyncio.CancelledError as exc:
                    cancellation = exc
            task.result()
        except Exception:
            if cancellation is not None:
                logger.exception("Late-interaction cleanup failed during cancellation.")
            raise
        finally:
            if cancellation is not None:
                raise cancellation

    async def _cleanup_flash_late_interaction(self, ctx: ScoringServeContext) -> None:
        query_keys = ctx.late_interaction_query_keys
        if not query_keys:
            return

        doc_keys = ctx.late_interaction_doc_keys or []

        try:
            # Stop documents before removing the query tensors they reference.
            # Also abort requests cancelled before add_request returned a collector.
            await self.engine_client.abort([*query_keys, *doc_keys])
        finally:
            # Bound release even when a timed-out abort enters this finally block.
            async with asyncio.timeout(_LATE_INTERACTION_CLEANUP_TIMEOUT_SECONDS):
                await self.engine_client.collective_rpc(
                    "release_late_interaction_query_cache",
                    args=(query_keys,),
                )

    async def _collect_late_interaction_batch(self, ctx: ScoringServeContext):
        generators = await self._prepare_generators(ctx)

        async def collect(generator: AsyncGenerator[PoolingRequestOutput, None]):
            final_result = None
            async for result in generator:
                if result.error is not None:
                    raise result.error.to_exception()
                final_result = result
            if final_result is None:
                raise ValueError("Failed to generate results for all prompts")
            return final_result

        tasks = [asyncio.create_task(collect(generator)) for generator in generators]
        batch_future = asyncio.gather(*tasks)
        try:
            results = await asyncio.shield(batch_future)
        except BaseException:
            # Stop and join every producer before aborting and releasing keys.
            for task in tasks:
                task.cancel()
            try:
                await self._await_late_interaction_cleanup(
                    asyncio.gather(batch_future, *tasks, return_exceptions=True)
                )
            except Exception:
                logger.exception("Failed to stop late-interaction producers.")
            raise
        ctx.final_res_batch = results

    async def _flash_late_interaction_encode_queries(self, ctx: ScoringServeContext):
        assert ctx.n_queries is not None
        assert ctx.engine_inputs is not None
        assert isinstance(ctx.pooling_params, PoolingParams)

        n_queries = ctx.n_queries
        n_docs = len(ctx.engine_inputs) - n_queries
        query_engine_inputs = ctx.engine_inputs[:n_queries]

        query_namespace = random_uuid()
        query_keys = [
            f"late-interaction-{query_namespace}-query-{i}" for i in range(n_queries)
        ]
        ctx.late_interaction_query_keys = query_keys
        query_uses = [n_docs if n_queries == 1 else 1] * n_queries

        for i in range(n_queries):
            pooling_params = ctx.pooling_params.clone()
            pooling_params.late_interaction_params = (
                build_late_interaction_query_params(
                    query_key=query_keys[i],
                    query_uses=query_uses[i],
                )
            )
            query_engine_inputs[i]["params"] = pooling_params

        assert n_queries == len(query_engine_inputs) == len(query_keys)

        query_ctx = ScoringServeContext(
            request=ctx.request,
            raw_request=ctx.raw_request,
            model_name=ctx.model_name,
            request_id=ctx.request_id,
            pooling_params=ctx.pooling_params,
            prompt_request_ids=query_keys,
            engine_inputs=query_engine_inputs,
            lora_request=ctx.lora_request,
            priorities=ctx.priorities,
            prompt_extras=ctx.prompt_extras,
        )

        await self._collect_late_interaction_batch(query_ctx)
        ctx.query_final_res_batch = query_ctx.final_res_batch

    async def _flash_late_interaction_encode_docs(self, ctx: ScoringServeContext):
        assert ctx.n_queries is not None
        assert ctx.engine_inputs is not None
        assert isinstance(ctx.pooling_params, PoolingParams)

        n_queries = ctx.n_queries
        n_docs = len(ctx.engine_inputs) - n_queries
        doc_engine_inputs = ctx.engine_inputs[n_queries:]

        query_keys = ctx.late_interaction_query_keys
        if query_keys is None:
            raise RuntimeError("Late-interaction query keys were not initialized.")
        doc_namespace = random_uuid()
        doc_keys = [f"late-interaction-{doc_namespace}-doc-{i}" for i in range(n_docs)]
        ctx.late_interaction_doc_keys = doc_keys

        for i in range(n_docs):
            query_idx = 0 if n_queries == 1 else i
            pooling_params = ctx.pooling_params.clone()
            pooling_params.late_interaction_params = build_late_interaction_doc_params(
                query_key=query_keys[query_idx]
            )
            doc_engine_inputs[i]["params"] = pooling_params

        assert n_docs == len(doc_engine_inputs) == len(doc_keys)

        doc_ctx = ScoringServeContext(
            request=ctx.request,
            raw_request=ctx.raw_request,
            model_name=ctx.model_name,
            request_id=ctx.request_id,
            pooling_params=ctx.pooling_params,
            prompt_request_ids=doc_keys,
            engine_inputs=doc_engine_inputs,
            lora_request=ctx.lora_request,
            priorities=ctx.priorities,
            prompt_extras=ctx.prompt_extras,
        )

        await self._collect_late_interaction_batch(doc_ctx)

        ctx.final_res_batch = doc_ctx.final_res_batch

_await_late_interaction_cleanup(task) async staticmethod

Bound cleanup waiting and defer cancellation until it finishes.

Source code in vllm/entrypoints/pooling/scoring/serving.py
@staticmethod
async def _await_late_interaction_cleanup(task: asyncio.Future[Any]) -> None:
    """Bound cleanup waiting and defer cancellation until it finishes."""
    loop = asyncio.get_running_loop()
    deadline = loop.time() + _LATE_INTERACTION_CLEANUP_TIMEOUT_SECONDS
    cancellation: asyncio.CancelledError | None = None
    try:
        while not task.done():
            remaining = deadline - loop.time()
            if remaining <= 0:
                task.cancel()
                task.add_done_callback(
                    lambda done: None if done.cancelled() else done.exception()
                )
                raise TimeoutError("Timed out waiting for late-interaction cleanup")
            try:
                await asyncio.wait({task}, timeout=remaining)
            except asyncio.CancelledError as exc:
                cancellation = exc
        task.result()
    except Exception:
        if cancellation is not None:
            logger.exception("Late-interaction cleanup failed during cancellation.")
        raise
    finally:
        if cancellation is not None:
            raise cancellation