qwenvl_oai.py 2.1 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859
  1. import copy
  2. import logging
  3. import os
  4. from pprint import pformat
  5. from typing import List
  6. from qwen_agent.llm import ModelServiceError
  7. from qwen_agent.llm.base import register_llm
  8. from qwen_agent.llm.oai import TextChatAtOAI
  9. from qwen_agent.llm.schema import ContentItem, Message
  10. from qwen_agent.log import logger
  11. from qwen_agent.utils.utils import encode_image_as_base64
  12. @register_llm('qwenvl_oai')
  13. class QwenVLChatAtOAI(TextChatAtOAI):
  14. @property
  15. def support_multimodal_input(self) -> bool:
  16. return True
  17. @staticmethod
  18. def convert_messages_to_dicts(messages: List[Message]) -> List[dict]:
  19. new_messages = []
  20. for msg in messages:
  21. content = msg.content
  22. if isinstance(content, str):
  23. content = [ContentItem(text=content)]
  24. assert isinstance(content, list)
  25. new_content = []
  26. for item in content:
  27. t, v = item.get_type_and_value()
  28. if t == 'text':
  29. new_content.append({'type': 'text', 'text': v})
  30. if t == 'image':
  31. if v.startswith('file://'):
  32. v = v[len('file://'):]
  33. if not v.startswith(('http://', 'https://', 'data:')):
  34. if os.path.exists(v):
  35. v = encode_image_as_base64(v, max_short_side_length=1080)
  36. else:
  37. raise ModelServiceError(f'Local image "{v}" does not exist.')
  38. new_content.append({'type': 'image_url', 'image_url': {'url': v}})
  39. new_msg = msg.model_dump()
  40. new_msg['content'] = new_content
  41. new_messages.append(new_msg)
  42. if logger.isEnabledFor(logging.DEBUG):
  43. lite_messages = copy.deepcopy(new_messages)
  44. for msg in lite_messages:
  45. for item in msg['content']:
  46. if item.get('image_url', {}).get('url', '').startswith('data:'):
  47. item['image_url']['url'] = item['image_url']['url'][:64] + '...'
  48. logger.debug(f'LLM Input:\n{pformat(lite_messages, indent=2)}')
  49. return new_messages