diff --git a/assistant/assistant.py b/assistant/assistant.py index 5be7ebe..1618b25 100644 --- a/assistant/assistant.py +++ b/assistant/assistant.py @@ -45,6 +45,7 @@ class Assistant(commands.Cog): enabled=True, chat_channel=None, model="gemma4", + vision_model="gemma4", temperature=0.7, system_prompt="You are a helpful assistant.", question_mode=False, @@ -96,7 +97,29 @@ class Assistant(commands.Cog): for mention in (f"<@{self.bot.user.id}>", f"<@!{self.bot.user.id}>"): content = content.replace(mention, "").strip() - if not content: + # Collect image URLs from this message and any replied-to message + image_urls = [] + for attachment in message.attachments: + if attachment.content_type and attachment.content_type.startswith("image/"): + image_urls.append(attachment.url) + + # Check replied-to message for images + if message.reference and message.reference.message_id: + try: + ref_msg = await message.channel.fetch_message(message.reference.message_id) + for attachment in ref_msg.attachments: + if attachment.content_type and attachment.content_type.startswith("image/"): + image_urls.append(attachment.url) + # Also check embeds for image URLs + for embed in ref_msg.embeds: + if embed.image and embed.image.url: + image_urls.append(embed.image.url) + if embed.thumbnail and embed.thumbnail.url: + image_urls.append(embed.thumbnail.url) + except (discord.NotFound, discord.HTTPException): + pass + + if not content and not image_urls: return if question_mode and not content.rstrip().endswith("?"): @@ -105,7 +128,7 @@ class Assistant(commands.Cog): return async with message.channel.typing(): - reply = await self._chat(guild, message.channel.id, content) + reply = await self._chat(guild, message.channel.id, content, image_urls=image_urls) if reply is None: await message.reply("No response from the assistant.") @@ -126,7 +149,7 @@ class Assistant(commands.Cog): # API # --------------------------------------------------------------------------- - async def _chat(self, guild: discord.Guild, channel_id: int, user_msg: str) -> str | None: + async def _chat(self, guild: discord.Guild, channel_id: int, user_msg: str, image_urls: list[str] | None = None) -> str | None: api_key = await self.config.api_key() if not api_key or not self.session: return None @@ -137,11 +160,27 @@ class Assistant(commands.Cog): max_tokens = await self.config.guild(guild).max_tokens() reasoning_effort = await self.config.guild(guild).reasoning_effort() + # Auto-switch to vision model when images are present + if image_urls: + vision_model = await self.config.guild(guild).vision_model() + if vision_model: + model = vision_model + key = (guild.id, channel_id) if key not in self.history: self.history[key] = [] - self.history[key].append({"role": "user", "content": user_msg}) + # Build user message content (text-only or multimodal) + if image_urls: + # OpenAI vision format: content is an array of parts + user_content = [] + if user_msg: + user_content.append({"type": "text", "text": user_msg}) + for url in image_urls: + user_content.append({"type": "image_url", "image_url": {"url": url}}) + self.history[key].append({"role": "user", "content": user_content}) + else: + self.history[key].append({"role": "user", "content": user_msg}) messages = [{"role": "system", "content": system_prompt}] + self.history[key][-20:] @@ -231,6 +270,20 @@ class Assistant(commands.Cog): await self.config.guild(ctx.guild).model.set(model) await ctx.send(f"Model set to **{model}**.") + @assistant.command(name="visionmodel") + async def assistant_visionmodel(self, ctx: commands.Context, model: str = None): + """Set the model used when images are detected. Omit to show current. + + Vision-capable models: gemma4, green-l, green-r, gpt-oss-120b, + gemma-3-27b-it, mistral-small-3.2-24b-instruct-2506, kimi-k2.6, kimi-k3 + """ + if model is None: + current = await self.config.guild(ctx.guild).vision_model() + await ctx.send(f"Vision model: **{current}**") + else: + await self.config.guild(ctx.guild).vision_model.set(model) + await ctx.send(f"Vision model set to **{model}**.") + @assistant.command(name="models") async def assistant_models(self, ctx: commands.Context): """List available chat models sorted by cost (cheapest first).""" @@ -414,6 +467,7 @@ class Assistant(commands.Cog): enabled = await cfg.enabled() chat_channel = await cfg.chat_channel() model = await cfg.model() + vision_model = await cfg.vision_model() temp = await cfg.temperature() sys_prompt = await cfg.system_prompt() q_mode = await cfg.question_mode() @@ -428,6 +482,7 @@ class Assistant(commands.Cog): embed.add_field(name="Chat Channel", value=channel_str, inline=True) embed.add_field(name="API Key", value="Set" if has_key else "Not set", inline=True) embed.add_field(name="Model", value=model, inline=True) + embed.add_field(name="Vision Model", value=vision_model, inline=True) embed.add_field(name="Temperature", value=str(temp), inline=True) embed.add_field(name="Question Mode", value=str(q_mode), inline=True) embed.add_field(name="Max Tokens", value=str(max_tokens) if max_tokens else "Unlimited", inline=True)