import json

def normalize_tokens(top_logprobs):
    # Create a dictionary to store unique tokens and their log probabilities
    unique_tokens = {}
    for token_info in top_logprobs:
        token = token_info['token'].lower()  # Normalize token to lowercase
        if token in unique_tokens:
            # If token already exists, update its log probability
            unique_tokens[token] = max(unique_tokens[token], token_info['logprob'])
        else:
            unique_tokens[token] = token_info['logprob']
    
    # Convert the dictionary back to a list of token information
    normalized_top_logprobs = []
    for token, logprob in unique_tokens.items():
        # Find the original token information with the maximum log probability
        max_logprob_token_info = max([t for t in top_logprobs if t['token'].lower() == token], key=lambda x: x['logprob'])
        normalized_top_logprobs.append({
            'token': max_logprob_token_info['token'],
            'logprob': logprob,
            'bytes': max_logprob_token_info['bytes']
        })
    
    return normalized_top_logprobs

# Example usage:
output = {
    "choices": [{
        "logprobs": {
            "content": [{
                "top_logprobs": [
                    {'token': 'none', 'logprob': -0.0012816318776458502, 'bytes': [110, 111, 110, 101]},
                    {'token': 'helper', 'logprob': -7.25128173828125, 'bytes': [104, 101, 108, 112, 101, 114]},
                    {'token': 'explanation', 'logprob': -7.75128173828125, 'bytes': [101, 120, 112, 108, 97, 110, 97, 116, 105, 111, 110]},
                    {'token': 'None', 'logprob': -9.87628173828125, 'bytes': [78, 111, 110, 101]},
                    {'token': 'none', 'logprob': -10.43878173828125, 'bytes': [110, 111, 110, 101]}
                ]
            }]
        }
    }]
}

normalized_top_logprobs = normalize_tokens(output["choices"][0]["logprobs"]["content"][0]["top_logprobs"])
print(json.dumps(normalized