vllm - 💡(How to fix) Fix [RFC]: Can we implement n-gram and suffix speculative decoding in model_runner_v2? [2 comments, 3 participants]

class ModelRunnerV2:
    def __init__(self, model):
        self.model = model

    def decode(self, input_text):
        # Call n-gram speculative decoding method
        n_gram_result = self.n_gram_speculative_decoding(input_text)
        
        # Call suffix speculative decoding method
        suffix_result = self.suffix_speculative_decoding(input_text)
        
        # Combine results
        return n_gram_result, suffix_result

    def n_gram_speculative_decoding(self, input_text):
        # Implement n-gram speculative decoding logic here
        # For example:
        n_grams = self.extract_n_grams(input_text)
        return self.model.predict(n_grams)

    def suffix_speculative_decoding(self, input_text):
        # Implement suffix speculative decoding logic here
        # For example:
        suffixes = self.extract_suffixes(input_text)
        return self.model.predict(suffixes)

    def extract_n_grams(self, input_text):
        # Implement n-gram extraction logic here
        # For example:
        n_grams = []
        for i in range(len(input_text)):
            n_grams.append(input_text[i:i+3])  # Extract 3-grams
        return n_grams

    def extract_suffixes(self, input_text):
        # Implement suffix extraction logic here
        # For example:
        suffixes = []
        for i in range(len(input_text)):
            suffixes.append(input_text[i:])  # Extract suffixes
        return suffixes