Koko210/miku-discord
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Implement comprehensive non-hierarchical logging system

Browse Source 
- Created new logging infrastructure with per-component filtering
- Added 6 log levels: DEBUG, INFO, API, WARNING, ERROR, CRITICAL
- Implemented non-hierarchical level control (any combination can be enabled)
- Migrated 917 print() statements across 31 files to structured logging
- Created web UI (system.html) for runtime configuration with dark theme
- Added global level controls to enable/disable levels across all components
- Added timestamp format control (off/time/date/datetime options)
- Implemented log rotation (10MB per file, 5 backups)
- Added API endpoints for dynamic log configuration
- Configured HTTP request logging with filtering via api.requests component
- Intercepted APScheduler logs with proper formatting
- Fixed persistence paths to use /app/memory for Docker volume compatibility
- Fixed checkbox display bug in web UI (enabled_levels now properly shown)
- Changed System Settings button to open in same tab instead of new window

Components: bot, api, api.requests, autonomous, persona, vision, llm,
conversation, mood, dm, scheduled, gpu, media, server, commands,
sentiment, core, apscheduler

All settings persist across container restarts via JSON config.
This commit is contained in:
koko210Serve
2026-01-10 20:46:19 +02:00
parent ce00f9bd95
commit 32c2a7b930
34 changed files with 2766 additions and 936 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
bot/utils/figurine_notifier.py
View File

@@ -10,7 +10,9 @@ import globals
from utils.twitter_fetcher import fetch_figurine_tweets_latest
from utils.image_handling import analyze_image_with_qwen, download_and_encode_image
from utils.llm import query_llama
from utils.logger import get_logger
logger = get_logger('bot')
from utils.dm_logger import dm_logger
@@ -37,14 +39,14 @@ def _ensure_dir(path: str) -> None:
def load_subscribers() -> List[int]:
try:
if os.path.exists(SUBSCRIBERS_FILE):
print(f"📁 Figurines: Loading subscribers from {SUBSCRIBERS_FILE}")
logger.debug(f"Loading subscribers from {SUBSCRIBERS_FILE}")
with open(SUBSCRIBERS_FILE, "r", encoding="utf-8") as f:
data = json.load(f)
subs = [int(uid) for uid in data.get("subscribers", [])]
print(f"📋 Figurines: Loaded {len(subs)} subscribers")
logger.debug(f"Loaded {len(subs)} subscribers")
return subs
except Exception as e:
print(f"⚠️ Failed to load figurine subscribers: {e}")
logger.error(f"Failed to load figurine subscribers: {e}")
return []
@@ -53,85 +55,85 @@ def save_subscribers(user_ids: List[int]) -> None:
_ensure_dir(SUBSCRIBERS_FILE)
# Save as strings to be JS-safe in the API layer if needed
payload = {"subscribers": [str(uid) for uid in user_ids]}
print(f"💾 Figurines: Saving {len(user_ids)} subscribers to {SUBSCRIBERS_FILE}")
logger.debug(f"Saving {len(user_ids)} subscribers to {SUBSCRIBERS_FILE}")
with open(SUBSCRIBERS_FILE, "w", encoding="utf-8") as f:
json.dump(payload, f, indent=2)
except Exception as e:
print(f"⚠️ Failed to save figurine subscribers: {e}")
logger.error(f"Failed to save figurine subscribers: {e}")
def add_subscriber(user_id: int) -> bool:
print(f"➕ Figurines: Adding subscriber {user_id}")
logger.info(f"Adding subscriber {user_id}")
subscribers = load_subscribers()
if user_id in subscribers:
print(f"ℹ️ Figurines: Subscriber {user_id} already present")
logger.info(f"Subscriber {user_id} already present")
return False
subscribers.append(user_id)
save_subscribers(subscribers)
print(f"✅ Figurines: Subscriber {user_id} added")
logger.info(f"Subscriber {user_id} added")
return True
def remove_subscriber(user_id: int) -> bool:
print(f"🗑️ Figurines: Removing subscriber {user_id}")
logger.info(f"Removing subscriber {user_id}")
subscribers = load_subscribers()
if user_id not in subscribers:
print(f"ℹ️ Figurines: Subscriber {user_id} was not present")
logger.info(f"Subscriber {user_id} was not present")
return False
subscribers = [uid for uid in subscribers if uid != user_id]
save_subscribers(subscribers)
print(f"✅ Figurines: Subscriber {user_id} removed")
logger.info(f"Subscriber {user_id} removed")
return True
def load_sent_tweets() -> List[str]:
try:
if os.path.exists(SENT_TWEETS_FILE):
print(f"📁 Figurines: Loading sent tweets from {SENT_TWEETS_FILE}")
logger.debug(f"Loading sent tweets from {SENT_TWEETS_FILE}")
with open(SENT_TWEETS_FILE, "r", encoding="utf-8") as f:
data = json.load(f)
urls = data.get("urls", [])
print(f"📋 Figurines: Loaded {len(urls)} sent tweet URLs")
logger.debug(f"Loaded {len(urls)} sent tweet URLs")
return urls
except Exception as e:
print(f"⚠️ Failed to load figurine sent tweets: {e}")
logger.error(f"Failed to load figurine sent tweets: {e}")
return []
def save_sent_tweets(urls: List[str]) -> None:
try:
_ensure_dir(SENT_TWEETS_FILE)
print(f"💾 Figurines: Saving {len(urls)} sent tweet URLs to {SENT_TWEETS_FILE}")
logger.debug(f"Saving {len(urls)} sent tweet URLs to {SENT_TWEETS_FILE}")
with open(SENT_TWEETS_FILE, "w", encoding="utf-8") as f:
json.dump({"urls": urls}, f, indent=2)
except Exception as e:
print(f"⚠️ Failed to save figurine sent tweets: {e}")
logger.error(f"Failed to save figurine sent tweets: {e}")
async def choose_random_figurine_tweet() -> Dict[str, Any] | None:
"""Fetch figurine tweets from multiple sources, filter out sent, and pick one randomly."""
print("🔎 Figurines: Fetching figurine tweets by Latest across sources…")
logger.info("Fetching figurine tweets by Latest across sources")
tweets = await fetch_figurine_tweets_latest(limit_per_source=10)
if not tweets:
print("📭 No figurine tweets found across sources")
logger.warning("No figurine tweets found across sources")
return None
sent_urls = set(load_sent_tweets())
fresh = [t for t in tweets if t.get("url") not in sent_urls]
print(f"🧮 Figurines: {len(tweets)} total, {len(fresh)} fresh after filtering sent")
logger.debug(f"{len(tweets)} total, {len(fresh)} fresh after filtering sent")
if not fresh:
print("ℹ️ All figurine tweets have been sent before; allowing reuse")
logger.warning("All figurine tweets have been sent before; allowing reuse")
fresh = tweets
chosen = random.choice(fresh)
print(f"🎯 Chosen figurine tweet: {chosen.get('url')}")
logger.info(f"Chosen figurine tweet: {chosen.get('url')}")
return chosen
async def send_figurine_dm_to_user(client: discord.Client, user_id: int, tweet: Dict[str, Any]) -> Tuple[bool, str]:
"""Send the figurine tweet to a single subscriber via DM, with analysis and LLM commentary."""
try:
print(f"✉️ Figurines: Preparing DM to user {user_id}")
logger.debug(f"Preparing DM to user {user_id}")
user = client.get_user(user_id)
if user is None:
# Try fetching
@@ -169,7 +171,7 @@ async def send_figurine_dm_to_user(client: discord.Client, user_id: int, tweet:
img_desc = await analyze_image_with_qwen(base64_img)
base_prompt += f"\n\nImage looks like: {img_desc}"
except Exception as e:
print(f"⚠️ Image analysis failed: {e}")
logger.warning(f"Image analysis failed: {e}")
# Include tweet text too
tweet_text = tweet.get("text", "").strip()
@@ -190,14 +192,14 @@ async def send_figurine_dm_to_user(client: discord.Client, user_id: int, tweet:
# Send the tweet URL first (convert to fxtwitter for better embeds)
fx_tweet_url = convert_to_fxtwitter(tweet_url)
tweet_message = await dm.send(fx_tweet_url)
print(f"✅ Figurines: Tweet URL sent to {user_id}: {fx_tweet_url}")
logger.info(f"Tweet URL sent to {user_id}: {fx_tweet_url}")
# Log the tweet URL message
dm_logger.log_user_message(user, tweet_message, is_bot_message=True)
# Send Miku's comment
comment_message = await dm.send(miku_comment)
print(f"✅ Figurines: Miku comment sent to {user_id}")
logger.info(f"Miku comment sent to {user_id}")
# Log the comment message
dm_logger.log_user_message(user, comment_message, is_bot_message=True)
@@ -212,27 +214,27 @@ async def send_figurine_dm_to_user(client: discord.Client, user_id: int, tweet:
# Use empty user prompt since this was initiated by Miku
globals.conversation_history.setdefault(user_id_str, []).append((tweet_context, miku_comment))
print(f"📝 Figurines: Messages logged to both DM history and conversation context for user {user_id}")
logger.debug(f"Messages logged to both DM history and conversation context for user {user_id}")
return True, "ok"
except Exception as e:
print(f"❌ Figurines: Failed DM to {user_id}: {e}")
logger.error(f"Failed DM to {user_id}: {e}")
return False, f"{e}"
async def send_figurine_dm_to_single_user(client: discord.Client, user_id: int, tweet_url: str = None) -> Dict[str, Any]:
"""Send a figurine tweet to a single user, either from search or specific URL."""
print(f"🎯 Figurines: Sending DM to single user {user_id}")
logger.info(f"Sending DM to single user {user_id}")
if tweet_url:
# Use specific tweet URL
print(f"📎 Figurines: Using specific tweet URL: {tweet_url}")
logger.info(f"Using specific tweet URL: {tweet_url}")
tweet = await fetch_specific_tweet_by_url(tweet_url)
if not tweet:
return {"status": "error", "message": "Failed to fetch specified tweet"}
else:
# Search for a random tweet
print("🔎 Figurines: Searching for random figurine tweet")
logger.info("Searching for random figurine tweet")
tweet = await choose_random_figurine_tweet()
if not tweet:
return {"status": "error", "message": "No figurine tweets found"}
@@ -256,7 +258,7 @@ async def send_figurine_dm_to_single_user(client: discord.Client, user_id: int,
"failed": [],
"tweet": {"url": tweet.get("url", ""), "username": tweet.get("username", "")}
}
print(f"✅ Figurines: Single user DM sent successfully → {result}")
logger.info(f"Single user DM sent successfully → {result}")
return result
else:
result = {
@@ -265,27 +267,27 @@ async def send_figurine_dm_to_single_user(client: discord.Client, user_id: int,
"failed": [{"user_id": str(user_id), "error": msg}],
"message": f"Failed to send DM: {msg}"
}
print(f"❌ Figurines: Single user DM failed → {result}")
logger.error(f"Single user DM failed → {result}")
return result
async def fetch_specific_tweet_by_url(tweet_url: str) -> Dict[str, Any] | None:
"""Fetch a specific tweet by URL for manual figurine notifications."""
try:
print(f"🔗 Figurines: Fetching specific tweet from URL: {tweet_url}")
logger.debug(f"Fetching specific tweet from URL: {tweet_url}")
# Extract tweet ID from URL
tweet_id = None
if "/status/" in tweet_url:
try:
tweet_id = tweet_url.split("/status/")[1].split("?")[0].split("/")[0]
print(f"📋 Figurines: Extracted tweet ID: {tweet_id}")
logger.debug(f"Extracted tweet ID: {tweet_id}")
except Exception as e:
print(f"❌ Figurines: Failed to extract tweet ID from URL: {e}")
logger.error(f"Failed to extract tweet ID from URL: {e}")
return None
if not tweet_id:
print("❌ Figurines: Could not extract tweet ID from URL")
logger.error("Could not extract tweet ID from URL")
return None
# Set up twscrape API (same pattern as existing functions)
@@ -313,15 +315,15 @@ async def fetch_specific_tweet_by_url(tweet_url: str) -> Dict[str, Any] | None:
# Try to fetch the tweet using search instead of tweet_details
# Search for the specific tweet ID should return it if accessible
print(f"🔍 Figurines: Searching for tweet with ID {tweet_id}")
logger.debug(f"Searching for tweet with ID {tweet_id}")
search_results = []
try:
# Search using the tweet ID - this should find the specific tweet
from twscrape import gather
search_results = await gather(api.search(f"{tweet_id}", limit=1))
print(f"🔍 Figurines: Search returned {len(search_results)} results")
logger.debug(f"Search returned {len(search_results)} results")
except Exception as search_error:
print(f"⚠️ Figurines: Search failed: {search_error}")
logger.warning(f"Search failed: {search_error}")
return None
# Check if we found the tweet
@@ -329,21 +331,21 @@ async def fetch_specific_tweet_by_url(tweet_url: str) -> Dict[str, Any] | None:
for tweet in search_results:
if str(tweet.id) == str(tweet_id):
tweet_data = tweet
print(f"✅ Figurines: Found matching tweet with ID {tweet.id}")
logger.debug(f"Found matching tweet with ID {tweet.id}")
break
if not tweet_data and search_results:
# If no exact match but we have results, use the first one
tweet_data = search_results[0]
print(f"🔍 Figurines: Using first search result with ID {tweet_data.id}")
logger.debug(f"Using first search result with ID {tweet_data.id}")
if tweet_data:
# Extract data using the same pattern as the working search code
username = tweet_data.user.username if hasattr(tweet_data, 'user') and tweet_data.user else "unknown"
text_content = tweet_data.rawContent if hasattr(tweet_data, 'rawContent') else ""
print(f"🔍 Figurines: Found tweet from @{username}")
print(f"🔍 Figurines: Tweet text: {text_content[:100]}...")
logger.debug(f"Found tweet from @{username}")
logger.debug(f"Tweet text: {text_content[:100]}...")
# For media, we'll need to extract it from the tweet_url using the same method as other functions
# But for now, let's see if we can get basic tweet data working first
@@ -354,37 +356,37 @@ async def fetch_specific_tweet_by_url(tweet_url: str) -> Dict[str, Any] | None:
"media": [] # We'll add media extraction later
}
print(f"✅ Figurines: Successfully fetched tweet from @{result['username']}")
logger.info(f"Successfully fetched tweet from @{result['username']}")
return result
else:
print("❌ Figurines: No tweet found with the specified ID")
logger.error("No tweet found with the specified ID")
return None
except Exception as e:
print(f"❌ Figurines: Error fetching tweet by URL: {e}")
logger.error(f"Error fetching tweet by URL: {e}")
return None
async def send_figurine_dm_to_all_subscribers(client: discord.Client, tweet_url: str = None) -> Dict[str, Any]:
"""Pick a figurine tweet and DM it to all subscribers, recording the sent URL."""
print("🚀 Figurines: Sending figurine DM to all subscribers…")
logger.info("Sending figurine DM to all subscribers")
subscribers = load_subscribers()
if not subscribers:
print("ℹ️ Figurines: No subscribers configured")
logger.warning("No subscribers configured")
return {"status": "no_subscribers"}
if tweet_url:
# Use specific tweet URL
print(f"📎 Figurines: Using specific tweet URL for all subscribers: {tweet_url}")
logger.info(f"Using specific tweet URL for all subscribers: {tweet_url}")
tweet = await fetch_specific_tweet_by_url(tweet_url)
if not tweet:
print("ℹ️ Figurines: Failed to fetch specified tweet")
logger.warning("Failed to fetch specified tweet")
return {"status": "no_tweet", "message": "Failed to fetch specified tweet"}
else:
# Search for random tweet
tweet = await choose_random_figurine_tweet()
if tweet is None:
print("ℹ️ Figurines: No tweet to send")
logger.warning("No tweet to send")
return {"status": "no_tweet"}
results = {"sent": [], "failed": []}
@@ -393,7 +395,7 @@ async def send_figurine_dm_to_all_subscribers(client: discord.Client, tweet_url:
if ok:
results["sent"].append(str(uid))
else:
print(f"⚠️ Failed to DM user {uid}: {msg}")
logger.warning(f"Failed to DM user {uid}: {msg}")
results["failed"].append({"user_id": str(uid), "error": msg})
# Record as sent if at least one success to avoid repeats
@@ -407,7 +409,7 @@ async def send_figurine_dm_to_all_subscribers(client: discord.Client, tweet_url:
save_sent_tweets(sent_urls)
summary = {"status": "ok", **results, "tweet": {"url": tweet.get("url", ""), "username": tweet.get("username", "")}}
print(f"📦 Figurines: DM send complete → {summary}")
logger.info(f"DM send complete → {summary}")
return summary